{ "architecture": "block", "artifacts": { "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" }, "batch_size": 32, "canonical_sha256_without_self": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515", "depth": 32, "diagnostics": [ { "forward_identity_gate": { "comparisons": { "detached_learned": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_value_backward": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "initialization_negative_control": { "comparisons": { "detached_learned": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "uniform_value_backward": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } } }, "passed": true }, "loss_scale_gate": { "checks": { "attention_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "mlp_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_attention_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_mlp_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_attention_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_mlp_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } } }, "passed": true }, "modes": { "detached_learned": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } } }, "learned": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } } }, "uniform_value_backward": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_value_backward", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } } } }, "step": 0 }, { "modes": { "learned": { "bits_per_byte": 5.150682269327325, "forward": { "activation_sha256": { "attention_branch_output": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", "mlp_branch_output": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", "post_attention_state": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", "post_mlp_state": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", "pre_attention_input": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", "pre_mlp_input": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f" }, "logits_sha256": "bdadcb0e52fa595647848016703b410d7bcf45d1b7a624e8a3f61c3e1956509d", "loss_nats": 3.570180892944336, "mixer_summary_sha256": "023101d1bfcf41624a0444a7f14bed4c44082764bd9047e90be0fb0a456542ad" }, "loss_nats": 3.570180892944336, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6930912733078003, "group": 1, "latest_source_mass": 0.49764734506607056, "layer": 1, "max_source_mass": 0.5023525953292847, "mixer_index": 1, "normalized_entropy": 0.9999193428845807, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5023525953292847, "median": 0.5031890273094177, "p05": 0.4936259984970093, "p95": 0.5090169310569763 }, { "label": "current_group_partial", "mean": 0.49764734506607056, "median": 0.4968109726905823, "p05": 0.4909830391407013, "p95": 0.5063740015029907 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.692903995513916, "group": 1, "latest_source_mass": 0.48966121673583984, "layer": 2, "max_source_mass": 0.5103387832641602, "mixer_index": 2, "normalized_entropy": 0.9996491581400753, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5103387832641602, "median": 0.5101123452186584, "p05": 0.5045775175094604, "p95": 0.5169917345046997 }, { "label": "current_group_partial", "mean": 0.48966121673583984, "median": 0.48988765478134155, "p05": 0.4830082654953003, "p95": 0.49542248249053955 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6928629875183105, "group": 1, "latest_source_mass": 0.4946908950805664, "layer": 2, "max_source_mass": 0.5053091049194336, "mixer_index": 3, "normalized_entropy": 0.9995899961081784, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5053091049194336, "median": 0.5084525346755981, "p05": 0.4841057062149048, "p95": 0.5181366801261902 }, { "label": "current_group_partial", "mean": 0.4946908950805664, "median": 0.49154743552207947, "p05": 0.4818633496761322, "p95": 0.5158942937850952 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6927823424339294, "group": 1, "latest_source_mass": 0.4873165488243103, "layer": 3, "max_source_mass": 0.5126834511756897, "mixer_index": 4, "normalized_entropy": 0.9994736498448697, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5126834511756897, "median": 0.5128336548805237, "p05": 0.5050104260444641, "p95": 0.5203934907913208 }, { "label": "current_group_partial", "mean": 0.4873165488243103, "median": 0.4871663451194763, "p05": 0.4796065092086792, "p95": 0.4949895739555359 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6928155422210693, "group": 1, "latest_source_mass": 0.49704670906066895, "layer": 3, "max_source_mass": 0.502953290939331, "mixer_index": 5, "normalized_entropy": 0.999521547013135, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.502953290939331, "median": 0.5012187957763672, "p05": 0.4864824414253235, "p95": 0.5226666927337646 }, { "label": "current_group_partial", "mean": 0.49704670906066895, "median": 0.4987812042236328, "p05": 0.4773333668708801, "p95": 0.5135174989700317 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.692751407623291, "group": 1, "latest_source_mass": 0.48708435893058777, "layer": 4, "max_source_mass": 0.5129156112670898, "mixer_index": 6, "normalized_entropy": 0.9994290203469708, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5129156112670898, "median": 0.512997567653656, "p05": 0.504144549369812, "p95": 0.5216691493988037 }, { "label": "current_group_partial", "mean": 0.48708435893058777, "median": 0.4870024621486664, "p05": 0.4783308207988739, "p95": 0.4958554208278656 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6925709247589111, "group": 1, "latest_source_mass": 0.48873060941696167, "layer": 4, "max_source_mass": 0.5112694501876831, "mixer_index": 7, "normalized_entropy": 0.9991686386135645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5112694501876831, "median": 0.5127402544021606, "p05": 0.4926953613758087, "p95": 0.5298716425895691 }, { "label": "current_group_partial", "mean": 0.48873060941696167, "median": 0.48725977540016174, "p05": 0.4701283872127533, "p95": 0.5073046088218689 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6929247379302979, "group": 2, "latest_source_mass": 0.49048691987991333, "layer": 5, "max_source_mass": 0.5095131397247314, "mixer_index": 8, "normalized_entropy": 0.9996790831213254, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5095131397247314, "median": 0.5093363523483276, "p05": 0.502471923828125, "p95": 0.5177270770072937 }, { "label": "completed_group_1", "mean": 0.49048691987991333, "median": 0.49066367745399475, "p05": 0.4822728931903839, "p95": 0.4975281059741974 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0983518362045288, "group": 2, "latest_source_mass": 0.33183109760284424, "layer": 5, "max_source_mass": 0.33983534574508667, "mixer_index": 9, "normalized_entropy": 0.999762925950977, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.33983534574508667, "median": 0.3411724865436554, "p05": 0.32688987255096436, "p95": 0.3497551679611206 }, { "label": "completed_group_1", "mean": 0.32833361625671387, "median": 0.328115850687027, "p05": 0.31867384910583496, "p95": 0.33767229318618774 }, { "label": "current_group_partial", "mean": 0.33183109760284424, "median": 0.331381618976593, "p05": 0.3268507122993469, "p95": 0.33714187145233154 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098447322845459, "group": 2, "latest_source_mass": 0.3288182318210602, "layer": 6, "max_source_mass": 0.340813547372818, "mixer_index": 10, "normalized_entropy": 0.9998498416371705, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.340813547372818, "median": 0.340879887342453, "p05": 0.33481645584106445, "p95": 0.34668514132499695 }, { "label": "completed_group_1", "mean": 0.3303682208061218, "median": 0.33054590225219727, "p05": 0.325665682554245, "p95": 0.33449190855026245 }, { "label": "current_group_partial", "mean": 0.3288182318210602, "median": 0.3287218511104584, "p05": 0.3254890441894531, "p95": 0.33243927359580994 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0980432033538818, "group": 2, "latest_source_mass": 0.32141047716140747, "layer": 6, "max_source_mass": 0.34726741909980774, "mixer_index": 11, "normalized_entropy": 0.9994819962236925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34726741909980774, "median": 0.34668511152267456, "p05": 0.3404209613800049, "p95": 0.35565274953842163 }, { "label": "completed_group_1", "mean": 0.3313220739364624, "median": 0.3316044509410858, "p05": 0.3264642655849457, "p95": 0.33527836203575134 }, { "label": "current_group_partial", "mean": 0.32141047716140747, "median": 0.321965754032135, "p05": 0.3149893283843994, "p95": 0.326220840215683 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0985240936279297, "group": 2, "latest_source_mass": 0.32844245433807373, "layer": 7, "max_source_mass": 0.3373557925224304, "mixer_index": 12, "normalized_entropy": 0.9999197214148341, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3373557925224304, "median": 0.33718767762184143, "p05": 0.33250534534454346, "p95": 0.34282490611076355 }, { "label": "completed_group_1", "mean": 0.33420175313949585, "median": 0.3341767191886902, "p05": 0.3307935893535614, "p95": 0.3377305567264557 }, { "label": "current_group_partial", "mean": 0.32844245433807373, "median": 0.3287346363067627, "p05": 0.3250963091850281, "p95": 0.33099210262298584 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.098268985748291, "group": 2, "latest_source_mass": 0.32483887672424316, "layer": 7, "max_source_mass": 0.3445510268211365, "mixer_index": 13, "normalized_entropy": 0.9996875122157655, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3445510268211365, "median": 0.3445873260498047, "p05": 0.3388661742210388, "p95": 0.3502572774887085 }, { "label": "completed_group_1", "mean": 0.330610066652298, "median": 0.330571711063385, "p05": 0.3262380361557007, "p95": 0.3352304697036743 }, { "label": "current_group_partial", "mean": 0.32483887672424316, "median": 0.32478615641593933, "p05": 0.3213251233100891, "p95": 0.3284890055656433 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0985302925109863, "group": 2, "latest_source_mass": 0.3295760750770569, "layer": 8, "max_source_mass": 0.3371829390525818, "mixer_index": 14, "normalized_entropy": 0.9999253638813536, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3371829390525818, "median": 0.33710986375808716, "p05": 0.330502450466156, "p95": 0.34345686435699463 }, { "label": "completed_group_1", "mean": 0.3332410156726837, "median": 0.33316272497177124, "p05": 0.32938438653945923, "p95": 0.33757662773132324 }, { "label": "current_group_partial", "mean": 0.3295760750770569, "median": 0.3295734226703644, "p05": 0.3265466094017029, "p95": 0.3326798677444458 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0983161926269531, "group": 2, "latest_source_mass": 0.32498204708099365, "layer": 8, "max_source_mass": 0.3423102796077728, "mixer_index": 15, "normalized_entropy": 0.9997304817684903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3423102796077728, "median": 0.34171032905578613, "p05": 0.3346346318721771, "p95": 0.35120290517807007 }, { "label": "completed_group_1", "mean": 0.3327076733112335, "median": 0.3330804705619812, "p05": 0.32750606536865234, "p95": 0.33743682503700256 }, { "label": "current_group_partial", "mean": 0.32498204708099365, "median": 0.3252435028553009, "p05": 0.31977349519729614, "p95": 0.3294060230255127 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098489761352539, "group": 3, "latest_source_mass": 0.327616423368454, "layer": 9, "max_source_mass": 0.33793729543685913, "mixer_index": 16, "normalized_entropy": 0.9998884708310343, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.33793729543685913, "median": 0.3376197814941406, "p05": 0.3321118652820587, "p95": 0.34503501653671265 }, { "label": "completed_group_1", "mean": 0.3344462513923645, "median": 0.3347727060317993, "p05": 0.33071762323379517, "p95": 0.33719509840011597 }, { "label": "completed_group_2", "mean": 0.327616423368454, "median": 0.3275512754917145, "p05": 0.32320931553840637, "p95": 0.33203738927841187 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3861088752746582, "group": 3, "latest_source_mass": 0.2466907799243927, "layer": 9, "max_source_mass": 0.2561652958393097, "mixer_index": 17, "normalized_entropy": 0.9998662002454641, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2561652958393097, "median": 0.25643306970596313, "p05": 0.25058528780937195, "p95": 0.2609494626522064 }, { "label": "completed_group_1", "mean": 0.2513129711151123, "median": 0.2511752247810364, "p05": 0.24789966642856598, "p95": 0.25490516424179077 }, { "label": "completed_group_2", "mean": 0.2458309531211853, "median": 0.2456333041191101, "p05": 0.2419084906578064, "p95": 0.2499997466802597 }, { "label": "current_group_partial", "mean": 0.2466907799243927, "median": 0.246699720621109, "p05": 0.24314038455486298, "p95": 0.25063008069992065 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3861844539642334, "group": 3, "latest_source_mass": 0.24695077538490295, "layer": 10, "max_source_mass": 0.2536832094192505, "mixer_index": 18, "normalized_entropy": 0.9999207187457876, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2536832094192505, "median": 0.253880113363266, "p05": 0.24814625084400177, "p95": 0.2593393921852112 }, { "label": "completed_group_1", "mean": 0.25214672088623047, "median": 0.2522420883178711, "p05": 0.24960598349571228, "p95": 0.2543295919895172 }, { "label": "completed_group_2", "mean": 0.24721932411193848, "median": 0.24704834818840027, "p05": 0.24384759366512299, "p95": 0.25051552057266235 }, { "label": "current_group_partial", "mean": 0.24695077538490295, "median": 0.24677348136901855, "p05": 0.24500520527362823, "p95": 0.2492748200893402 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3860987424850464, "group": 3, "latest_source_mass": 0.24684447050094604, "layer": 10, "max_source_mass": 0.25705933570861816, "mixer_index": 19, "normalized_entropy": 0.9998588909828025, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25705933570861816, "median": 0.2568470239639282, "p05": 0.2513251304626465, "p95": 0.26331475377082825 }, { "label": "completed_group_1", "mean": 0.24983704090118408, "median": 0.24969333410263062, "p05": 0.2465076744556427, "p95": 0.25355273485183716 }, { "label": "completed_group_2", "mean": 0.2462591528892517, "median": 0.24622508883476257, "p05": 0.24337750673294067, "p95": 0.24924828112125397 }, { "label": "current_group_partial", "mean": 0.24684447050094604, "median": 0.24683824181556702, "p05": 0.24403956532478333, "p95": 0.24949496984481812 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862221240997314, "group": 3, "latest_source_mass": 0.24620777368545532, "layer": 11, "max_source_mass": 0.25285202264785767, "mixer_index": 20, "normalized_entropy": 0.9999478920046239, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2514326572418213, "median": 0.25176799297332764, "p05": 0.24684108793735504, "p95": 0.2554701566696167 }, { "label": "completed_group_1", "mean": 0.25285202264785767, "median": 0.25268346071243286, "p05": 0.2510001063346863, "p95": 0.2550324499607086 }, { "label": "completed_group_2", "mean": 0.24950754642486572, "median": 0.2494041621685028, "p05": 0.2475442737340927, "p95": 0.2515066862106323 }, { "label": "current_group_partial", "mean": 0.24620777368545532, "median": 0.24642038345336914, "p05": 0.2445429414510727, "p95": 0.2473805695772171 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.386016845703125, "group": 3, "latest_source_mass": 0.24290117621421814, "layer": 11, "max_source_mass": 0.25482746958732605, "mixer_index": 21, "normalized_entropy": 0.999799814942231, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25482746958732605, "median": 0.2545710802078247, "p05": 0.2466055005788803, "p95": 0.2635689973831177 }, { "label": "completed_group_1", "mean": 0.2536851167678833, "median": 0.2534823715686798, "p05": 0.2508774995803833, "p95": 0.2568957209587097 }, { "label": "completed_group_2", "mean": 0.24858617782592773, "median": 0.24861112236976624, "p05": 0.24390698969364166, "p95": 0.252436101436615 }, { "label": "current_group_partial", "mean": 0.24290117621421814, "median": 0.24232372641563416, "p05": 0.23918122053146362, "p95": 0.24772994220256805 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862277269363403, "group": 3, "latest_source_mass": 0.24656552076339722, "layer": 12, "max_source_mass": 0.2530350983142853, "mixer_index": 22, "normalized_entropy": 0.9999519335969191, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25171470642089844, "median": 0.2513781189918518, "p05": 0.24833296239376068, "p95": 0.25546184182167053 }, { "label": "completed_group_1", "mean": 0.2530350983142853, "median": 0.25303736329078674, "p05": 0.25148919224739075, "p95": 0.2544969618320465 }, { "label": "completed_group_2", "mean": 0.24868467450141907, "median": 0.24892255663871765, "p05": 0.2467103898525238, "p95": 0.25010940432548523 }, { "label": "current_group_partial", "mean": 0.24656552076339722, "median": 0.24677836894989014, "p05": 0.24485518038272858, "p95": 0.2477675825357437 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3859888315200806, "group": 3, "latest_source_mass": 0.24197682738304138, "layer": 12, "max_source_mass": 0.2546530067920685, "mixer_index": 23, "normalized_entropy": 0.9997796069807546, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25458407402038574, "median": 0.2539653182029724, "p05": 0.24643409252166748, "p95": 0.26325899362564087 }, { "label": "completed_group_1", "mean": 0.2546530067920685, "median": 0.2548711895942688, "p05": 0.25055474042892456, "p95": 0.25881263613700867 }, { "label": "completed_group_2", "mean": 0.248786062002182, "median": 0.24910664558410645, "p05": 0.2454056739807129, "p95": 0.2517758011817932 }, { "label": "current_group_partial", "mean": 0.24197682738304138, "median": 0.2416902482509613, "p05": 0.23907747864723206, "p95": 0.2456972599029541 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862541913986206, "group": 4, "latest_source_mass": 0.24767187237739563, "layer": 13, "max_source_mass": 0.2521768808364868, "mixer_index": 24, "normalized_entropy": 0.999971023671165, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2501963973045349, "median": 0.250087708234787, "p05": 0.24644337594509125, "p95": 0.25403982400894165 }, { "label": "completed_group_1", "mean": 0.2521768808364868, "median": 0.2521744668483734, "p05": 0.25072887539863586, "p95": 0.25345146656036377 }, { "label": "completed_group_2", "mean": 0.24995486438274384, "median": 0.2498224377632141, "p05": 0.2482302337884903, "p95": 0.2518054246902466 }, { "label": "completed_group_3", "mean": 0.24767187237739563, "median": 0.2475992739200592, "p05": 0.24603278934955597, "p95": 0.2497682422399521 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6090729236602783, "group": 4, "latest_source_mass": 0.19462624192237854, "layer": 13, "max_source_mass": 0.20631682872772217, "mixer_index": 25, "normalized_entropy": 0.9997732197241024, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20360878109931946, "median": 0.20399323105812073, "p05": 0.19613273441791534, "p95": 0.21039927005767822 }, { "label": "completed_group_1", "mean": 0.20631682872772217, "median": 0.20582494139671326, "p05": 0.20368342101573944, "p95": 0.2099049985408783 }, { "label": "completed_group_2", "mean": 0.19986985623836517, "median": 0.19948531687259674, "p05": 0.19628161191940308, "p95": 0.20406028628349304 }, { "label": "completed_group_3", "mean": 0.19557830691337585, "median": 0.19561558961868286, "p05": 0.1910029649734497, "p95": 0.20039188861846924 }, { "label": "current_group_partial", "mean": 0.19462624192237854, "median": 0.1954103410243988, "p05": 0.19068367779254913, "p95": 0.19781020283699036 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093083620071411, "group": 4, "latest_source_mass": 0.1976698487997055, "layer": 14, "max_source_mass": 0.20362424850463867, "mixer_index": 26, "normalized_entropy": 0.9999195057939432, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2017374038696289, "median": 0.2026747167110443, "p05": 0.19666710495948792, "p95": 0.20696133375167847 }, { "label": "completed_group_1", "mean": 0.20362424850463867, "median": 0.20354315638542175, "p05": 0.20209211111068726, "p95": 0.20522044599056244 }, { "label": "completed_group_2", "mean": 0.20011675357818604, "median": 0.1997961699962616, "p05": 0.19781088829040527, "p95": 0.2023434340953827 }, { "label": "completed_group_3", "mean": 0.1968517303466797, "median": 0.19661611318588257, "p05": 0.19452345371246338, "p95": 0.19933368265628815 }, { "label": "current_group_partial", "mean": 0.1976698487997055, "median": 0.19780653715133667, "p05": 0.19629782438278198, "p95": 0.19887655973434448 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.608644723892212, "group": 4, "latest_source_mass": 0.18768835067749023, "layer": 14, "max_source_mass": 0.20831988751888275, "mixer_index": 27, "normalized_entropy": 0.9995071642492324, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20431089401245117, "median": 0.20455676317214966, "p05": 0.19753149151802063, "p95": 0.21038773655891418 }, { "label": "completed_group_1", "mean": 0.20831988751888275, "median": 0.20877230167388916, "p05": 0.2030324637889862, "p95": 0.2128065973520279 }, { "label": "completed_group_2", "mean": 0.20422273874282837, "median": 0.2047901451587677, "p05": 0.20030567049980164, "p95": 0.20715001225471497 }, { "label": "completed_group_3", "mean": 0.19545812904834747, "median": 0.19552549719810486, "p05": 0.1923498660326004, "p95": 0.19825047254562378 }, { "label": "current_group_partial", "mean": 0.18768835067749023, "median": 0.18748146295547485, "p05": 0.18511706590652466, "p95": 0.19059684872627258 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.609345555305481, "group": 4, "latest_source_mass": 0.19806164503097534, "layer": 15, "max_source_mass": 0.20386312901973724, "mixer_index": 28, "normalized_entropy": 0.9999426152895332, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2002599537372589, "median": 0.20108085870742798, "p05": 0.19542044401168823, "p95": 0.20405235886573792 }, { "label": "completed_group_1", "mean": 0.20386312901973724, "median": 0.20386025309562683, "p05": 0.20271247625350952, "p95": 0.20498213171958923 }, { "label": "completed_group_2", "mean": 0.2001732587814331, "median": 0.19999440014362335, "p05": 0.1985137164592743, "p95": 0.2019573152065277 }, { "label": "completed_group_3", "mean": 0.1976420283317566, "median": 0.19760513305664062, "p05": 0.1957571804523468, "p95": 0.20000392198562622 }, { "label": "current_group_partial", "mean": 0.19806164503097534, "median": 0.19796663522720337, "p05": 0.1969158947467804, "p95": 0.19929495453834534 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6086196899414062, "group": 4, "latest_source_mass": 0.18893863260746002, "layer": 15, "max_source_mass": 0.21025606989860535, "mixer_index": 29, "normalized_entropy": 0.9994916097810468, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20515945553779602, "median": 0.2049580067396164, "p05": 0.19868221879005432, "p95": 0.21147878468036652 }, { "label": "completed_group_1", "mean": 0.21025606989860535, "median": 0.2101147472858429, "p05": 0.20689907670021057, "p95": 0.21399712562561035 }, { "label": "completed_group_2", "mean": 0.20179016888141632, "median": 0.20176216959953308, "p05": 0.199407696723938, "p95": 0.20402206480503082 }, { "label": "completed_group_3", "mean": 0.1938556730747223, "median": 0.19379490613937378, "p05": 0.19004228711128235, "p95": 0.1977127194404602 }, { "label": "current_group_partial", "mean": 0.18893863260746002, "median": 0.1887960582971573, "p05": 0.18639500439167023, "p95": 0.19145210087299347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.609370470046997, "group": 4, "latest_source_mass": 0.1981920301914215, "layer": 16, "max_source_mass": 0.20340535044670105, "mixer_index": 30, "normalized_entropy": 0.9999580956888227, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2002836912870407, "median": 0.20032534003257751, "p05": 0.19668522477149963, "p95": 0.20378869771957397 }, { "label": "completed_group_1", "mean": 0.20340535044670105, "median": 0.20332223176956177, "p05": 0.2024117112159729, "p95": 0.20457619428634644 }, { "label": "completed_group_2", "mean": 0.20010054111480713, "median": 0.1998724788427353, "p05": 0.19848622381687164, "p95": 0.20170091092586517 }, { "label": "completed_group_3", "mean": 0.1980184018611908, "median": 0.19809064269065857, "p05": 0.19669371843338013, "p95": 0.19908645749092102 }, { "label": "current_group_partial", "mean": 0.1981920301914215, "median": 0.19810855388641357, "p05": 0.1971922218799591, "p95": 0.19958281517028809 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6083989143371582, "group": 4, "latest_source_mass": 0.18735264241695404, "layer": 16, "max_source_mass": 0.21095217764377594, "mixer_index": 31, "normalized_entropy": 0.9993544341854289, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20363345742225647, "median": 0.20384372770786285, "p05": 0.19405215978622437, "p95": 0.21326100826263428 }, { "label": "completed_group_1", "mean": 0.21095217764377594, "median": 0.21146853268146515, "p05": 0.20567592978477478, "p95": 0.21522513031959534 }, { "label": "completed_group_2", "mean": 0.20392385125160217, "median": 0.20381325483322144, "p05": 0.20115600526332855, "p95": 0.2067122757434845 }, { "label": "completed_group_3", "mean": 0.19413790106773376, "median": 0.19391843676567078, "p05": 0.18873284757137299, "p95": 0.1996975690126419 }, { "label": "current_group_partial", "mean": 0.18735264241695404, "median": 0.18673613667488098, "p05": 0.1829182654619217, "p95": 0.1927962601184845 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093392372131348, "group": 5, "latest_source_mass": 0.19563516974449158, "layer": 17, "max_source_mass": 0.20331618189811707, "mixer_index": 32, "normalized_entropy": 0.9999386896380388, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20084109902381897, "median": 0.20074692368507385, "p05": 0.198189377784729, "p95": 0.20358796417713165 }, { "label": "completed_group_1", "mean": 0.20331618189811707, "median": 0.20337940752506256, "p05": 0.20178094506263733, "p95": 0.20489434897899628 }, { "label": "completed_group_2", "mean": 0.20119993388652802, "median": 0.20122915506362915, "p05": 0.20026232302188873, "p95": 0.2019881308078766 }, { "label": "completed_group_3", "mean": 0.19900763034820557, "median": 0.198960542678833, "p05": 0.19746547937393188, "p95": 0.20062875747680664 }, { "label": "completed_group_4", "mean": 0.19563516974449158, "median": 0.19558461010456085, "p05": 0.193980410695076, "p95": 0.19730384647846222 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7908520698547363, "group": 5, "latest_source_mass": 0.1625179648399353, "layer": 17, "max_source_mass": 0.17730244994163513, "mixer_index": 33, "normalized_entropy": 0.9994935707672249, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17160072922706604, "median": 0.1718936264514923, "p05": 0.16487310826778412, "p95": 0.17849747836589813 }, { "label": "completed_group_1", "mean": 0.17730244994163513, "median": 0.1778835654258728, "p05": 0.17289136350154877, "p95": 0.18126235902309418 }, { "label": "completed_group_2", "mean": 0.1690303534269333, "median": 0.16928547620773315, "p05": 0.16646793484687805, "p95": 0.17125293612480164 }, { "label": "completed_group_3", "mean": 0.16099892556667328, "median": 0.16061408817768097, "p05": 0.15715192258358002, "p95": 0.1656636893749237 }, { "label": "completed_group_4", "mean": 0.15854957699775696, "median": 0.15825147926807404, "p05": 0.15517033636569977, "p95": 0.1620505452156067 }, { "label": "current_group_partial", "mean": 0.1625179648399353, "median": 0.16315963864326477, "p05": 0.15939058363437653, "p95": 0.16489268839359283 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7916775941848755, "group": 5, "latest_source_mass": 0.16463731229305267, "layer": 18, "max_source_mass": 0.16981256008148193, "mixer_index": 34, "normalized_entropy": 0.9999543046683522, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16706335544586182, "median": 0.16679415106773376, "p05": 0.16531850397586823, "p95": 0.16906341910362244 }, { "label": "completed_group_1", "mean": 0.16981256008148193, "median": 0.1699288785457611, "p05": 0.1682884395122528, "p95": 0.17106440663337708 }, { "label": "completed_group_2", "mean": 0.16858771443367004, "median": 0.16860879957675934, "p05": 0.16782470047473907, "p95": 0.16934865713119507 }, { "label": "completed_group_3", "mean": 0.16527186334133148, "median": 0.16520482301712036, "p05": 0.1643052101135254, "p95": 0.16638660430908203 }, { "label": "completed_group_4", "mean": 0.16462719440460205, "median": 0.16459274291992188, "p05": 0.1638425886631012, "p95": 0.16539013385772705 }, { "label": "current_group_partial", "mean": 0.16463731229305267, "median": 0.16451048851013184, "p05": 0.1639435589313507, "p95": 0.16552197933197021 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7898297309875488, "group": 5, "latest_source_mass": 0.153368279337883, "layer": 18, "max_source_mass": 0.18101517856121063, "mixer_index": 35, "normalized_entropy": 0.9989229925815112, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.17284835875034332, "median": 0.17269954085350037, "p05": 0.1658562421798706, "p95": 0.18014436960220337 }, { "label": "completed_group_1", "mean": 0.18101517856121063, "median": 0.1814284771680832, "p05": 0.17416825890541077, "p95": 0.18707150220870972 }, { "label": "completed_group_2", "mean": 0.1735592633485794, "median": 0.17370939254760742, "p05": 0.17075185477733612, "p95": 0.17625606060028076 }, { "label": "completed_group_3", "mean": 0.16231875121593475, "median": 0.1621963381767273, "p05": 0.15821577608585358, "p95": 0.16650474071502686 }, { "label": "completed_group_4", "mean": 0.15689018368721008, "median": 0.15674838423728943, "p05": 0.15215492248535156, "p95": 0.1616627275943756 }, { "label": "current_group_partial", "mean": 0.153368279337883, "median": 0.1528519243001938, "p05": 0.15056025981903076, "p95": 0.15731072425842285 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917213439941406, "group": 5, "latest_source_mass": 0.16463561356067657, "layer": 19, "max_source_mass": 0.16885727643966675, "mixer_index": 36, "normalized_entropy": 0.9999787219018127, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1661592274904251, "median": 0.1661478579044342, "p05": 0.16439414024353027, "p95": 0.1678619235754013 }, { "label": "completed_group_1", "mean": 0.16885727643966675, "median": 0.16877895593643188, "p05": 0.16819745302200317, "p95": 0.1697411835193634 }, { "label": "completed_group_2", "mean": 0.16764025390148163, "median": 0.16765299439430237, "p05": 0.1669837087392807, "p95": 0.1682416945695877 }, { "label": "completed_group_3", "mean": 0.16661542654037476, "median": 0.1665019690990448, "p05": 0.1658560335636139, "p95": 0.16755254566669464 }, { "label": "completed_group_4", "mean": 0.16609220206737518, "median": 0.16612619161605835, "p05": 0.16555683314800262, "p95": 0.16653065383434296 }, { "label": "current_group_partial", "mean": 0.16463561356067657, "median": 0.16466060280799866, "p05": 0.16384081542491913, "p95": 0.16551899909973145 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7890949249267578, "group": 5, "latest_source_mass": 0.15094393491744995, "layer": 19, "max_source_mass": 0.18413972854614258, "mixer_index": 37, "normalized_entropy": 0.9985128895105295, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.17241930961608887, "median": 0.17162901163101196, "p05": 0.1626318097114563, "p95": 0.18299198150634766 }, { "label": "completed_group_1", "mean": 0.18413972854614258, "median": 0.1848709136247635, "p05": 0.17655718326568604, "p95": 0.19053134322166443 }, { "label": "completed_group_2", "mean": 0.17484498023986816, "median": 0.17485198378562927, "p05": 0.1718076914548874, "p95": 0.17815767228603363 }, { "label": "completed_group_3", "mean": 0.1621202528476715, "median": 0.16197144985198975, "p05": 0.15698590874671936, "p95": 0.16731801629066467 }, { "label": "completed_group_4", "mean": 0.15553176403045654, "median": 0.15512266755104065, "p05": 0.1502593755722046, "p95": 0.16059254109859467 }, { "label": "current_group_partial", "mean": 0.15094393491744995, "median": 0.15083211660385132, "p05": 0.1470862179994583, "p95": 0.1549530029296875 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917208671569824, "group": 5, "latest_source_mass": 0.16669265925884247, "layer": 20, "max_source_mass": 0.168935626745224, "mixer_index": 38, "normalized_entropy": 0.9999784557739275, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16663745045661926, "median": 0.16669239103794098, "p05": 0.16406413912773132, "p95": 0.16939817368984222 }, { "label": "completed_group_1", "mean": 0.168935626745224, "median": 0.16887718439102173, "p05": 0.16814815998077393, "p95": 0.1698606014251709 }, { "label": "completed_group_2", "mean": 0.16597259044647217, "median": 0.16582076251506805, "p05": 0.16482511162757874, "p95": 0.1670236438512802 }, { "label": "completed_group_3", "mean": 0.16514870524406433, "median": 0.165022611618042, "p05": 0.16397784650325775, "p95": 0.1662973016500473 }, { "label": "completed_group_4", "mean": 0.16661295294761658, "median": 0.16667577624320984, "p05": 0.16593852639198303, "p95": 0.16715332865715027 }, { "label": "current_group_partial", "mean": 0.16669265925884247, "median": 0.1666145920753479, "p05": 0.16592761874198914, "p95": 0.1674128621816635 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.789101004600525, "group": 5, "latest_source_mass": 0.14993149042129517, "layer": 20, "max_source_mass": 0.18365857005119324, "mixer_index": 39, "normalized_entropy": 0.9985162826410648, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.17229264974594116, "median": 0.1720556616783142, "p05": 0.16247707605361938, "p95": 0.18216519057750702 }, { "label": "completed_group_1", "mean": 0.18365857005119324, "median": 0.18395906686782837, "p05": 0.17734748125076294, "p95": 0.18972161412239075 }, { "label": "completed_group_2", "mean": 0.1747075319290161, "median": 0.1746184229850769, "p05": 0.17210862040519714, "p95": 0.1774953156709671 }, { "label": "completed_group_3", "mean": 0.1640234887599945, "median": 0.1635349988937378, "p05": 0.15958356857299805, "p95": 0.16904467344284058 }, { "label": "completed_group_4", "mean": 0.15538626909255981, "median": 0.15498605370521545, "p05": 0.14995241165161133, "p95": 0.16101743280887604 }, { "label": "current_group_partial", "mean": 0.14993149042129517, "median": 0.15014703571796417, "p05": 0.14564283192157745, "p95": 0.1527552753686905 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917447090148926, "group": 6, "latest_source_mass": 0.1661863625049591, "layer": 21, "max_source_mass": 0.16768339276313782, "mixer_index": 40, "normalized_entropy": 0.9999917621681839, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1658121943473816, "median": 0.1657177209854126, "p05": 0.16437731683254242, "p95": 0.16778677701950073 }, { "label": "completed_group_1", "mean": 0.16768339276313782, "median": 0.16760629415512085, "p05": 0.1666821986436844, "p95": 0.16875657439231873 }, { "label": "completed_group_2", "mean": 0.16629494726657867, "median": 0.16622786223888397, "p05": 0.1655496209859848, "p95": 0.16718709468841553 }, { "label": "completed_group_3", "mean": 0.1669958382844925, "median": 0.16695073246955872, "p05": 0.1664678305387497, "p95": 0.16760030388832092 }, { "label": "completed_group_4", "mean": 0.16702723503112793, "median": 0.16713355481624603, "p05": 0.16618871688842773, "p95": 0.16781742870807648 }, { "label": "completed_group_5", "mean": 0.1661863625049591, "median": 0.16608929634094238, "p05": 0.16534176468849182, "p95": 0.16705378890037537 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9436898231506348, "group": 6, "latest_source_mass": 0.13651785254478455, "layer": 21, "max_source_mass": 0.15856821835041046, "mixer_index": 41, "normalized_entropy": 0.9988589781980651, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14778707921504974, "median": 0.14752492308616638, "p05": 0.13982102274894714, "p95": 0.1569063663482666 }, { "label": "completed_group_1", "mean": 0.15856821835041046, "median": 0.1585446149110794, "p05": 0.15396656095981598, "p95": 0.16390106081962585 }, { "label": "completed_group_2", "mean": 0.1500256508588791, "median": 0.14993107318878174, "p05": 0.14748509228229523, "p95": 0.15272916853427887 }, { "label": "completed_group_3", "mean": 0.1408001333475113, "median": 0.14052921533584595, "p05": 0.13647492229938507, "p95": 0.14521925151348114 }, { "label": "completed_group_4", "mean": 0.13533657789230347, "median": 0.13547882437705994, "p05": 0.13068656623363495, "p95": 0.13946673274040222 }, { "label": "completed_group_5", "mean": 0.130964457988739, "median": 0.13209164142608643, "p05": 0.12633010745048523, "p95": 0.13486123085021973 }, { "label": "current_group_partial", "mean": 0.13651785254478455, "median": 0.1365797072649002, "p05": 0.13271403312683105, "p95": 0.14000433683395386 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458942413330078, "group": 6, "latest_source_mass": 0.14246684312820435, "layer": 22, "max_source_mass": 0.14393000304698944, "mixer_index": 42, "normalized_entropy": 0.9999918250478763, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.142832413315773, "median": 0.14290162920951843, "p05": 0.14148348569869995, "p95": 0.1440550833940506 }, { "label": "completed_group_1", "mean": 0.1429906189441681, "median": 0.14299973845481873, "p05": 0.14231431484222412, "p95": 0.14357265830039978 }, { "label": "completed_group_2", "mean": 0.14157448709011078, "median": 0.14162302017211914, "p05": 0.1408008486032486, "p95": 0.1421196460723877 }, { "label": "completed_group_3", "mean": 0.14285019040107727, "median": 0.14287817478179932, "p05": 0.1423148512840271, "p95": 0.14345195889472961 }, { "label": "completed_group_4", "mean": 0.14393000304698944, "median": 0.14395871758460999, "p05": 0.14325706660747528, "p95": 0.14449745416641235 }, { "label": "completed_group_5", "mean": 0.14335544407367706, "median": 0.14326418936252594, "p05": 0.14292818307876587, "p95": 0.14392802119255066 }, { "label": "current_group_partial", "mean": 0.14246684312820435, "median": 0.14247724413871765, "p05": 0.14211606979370117, "p95": 0.14281725883483887 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9425597190856934, "group": 6, "latest_source_mass": 0.12947803735733032, "layer": 22, "max_source_mass": 0.1615946888923645, "mixer_index": 43, "normalized_entropy": 0.9982782195923864, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1483290195465088, "median": 0.14828777313232422, "p05": 0.13995377719402313, "p95": 0.15759113430976868 }, { "label": "completed_group_1", "mean": 0.1615946888923645, "median": 0.16178560256958008, "p05": 0.15419340133666992, "p95": 0.16922280192375183 }, { "label": "completed_group_2", "mean": 0.15280696749687195, "median": 0.1528012454509735, "p05": 0.15010300278663635, "p95": 0.1558753252029419 }, { "label": "completed_group_3", "mean": 0.14239415526390076, "median": 0.14164817333221436, "p05": 0.1380041539669037, "p95": 0.14742238819599152 }, { "label": "completed_group_4", "mean": 0.13462956249713898, "median": 0.1342291682958603, "p05": 0.12952053546905518, "p95": 0.14025796949863434 }, { "label": "completed_group_5", "mean": 0.1307675838470459, "median": 0.13075318932533264, "p05": 0.12602169811725616, "p95": 0.13501127064228058 }, { "label": "current_group_partial", "mean": 0.12947803735733032, "median": 0.1295393407344818, "p05": 0.1264311969280243, "p95": 0.13163118064403534 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458987712860107, "group": 6, "latest_source_mass": 0.14346569776535034, "layer": 23, "max_source_mass": 0.14346569776535034, "mixer_index": 44, "normalized_entropy": 0.9999941529832156, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1426449418067932, "median": 0.14265665411949158, "p05": 0.14160317182540894, "p95": 0.1436314880847931 }, { "label": "completed_group_1", "mean": 0.14291533827781677, "median": 0.14293473958969116, "p05": 0.1422625482082367, "p95": 0.14348529279232025 }, { "label": "completed_group_2", "mean": 0.1416938751935959, "median": 0.14168253540992737, "p05": 0.14088502526283264, "p95": 0.14245855808258057 }, { "label": "completed_group_3", "mean": 0.14274153113365173, "median": 0.14274577796459198, "p05": 0.14230182766914368, "p95": 0.14324283599853516 }, { "label": "completed_group_4", "mean": 0.14325544238090515, "median": 0.1432678997516632, "p05": 0.14277629554271698, "p95": 0.14376230537891388 }, { "label": "completed_group_5", "mean": 0.1432831883430481, "median": 0.14325812458992004, "p05": 0.14287351071834564, "p95": 0.1437322348356247 }, { "label": "current_group_partial", "mean": 0.14346569776535034, "median": 0.1435171663761139, "p05": 0.142959862947464, "p95": 0.1437678039073944 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.941068410873413, "group": 6, "latest_source_mass": 0.1254161298274994, "layer": 23, "max_source_mass": 0.16456490755081177, "mixer_index": 45, "normalized_entropy": 0.9975118387741332, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14912807941436768, "median": 0.14865416288375854, "p05": 0.13610364496707916, "p95": 0.16207551956176758 }, { "label": "completed_group_1", "mean": 0.16456490755081177, "median": 0.16586321592330933, "p05": 0.15482422709465027, "p95": 0.17167139053344727 }, { "label": "completed_group_2", "mean": 0.15460123121738434, "median": 0.1543833166360855, "p05": 0.15155401825904846, "p95": 0.15836837887763977 }, { "label": "completed_group_3", "mean": 0.1425517499446869, "median": 0.14188045263290405, "p05": 0.13683314621448517, "p95": 0.1494884341955185 }, { "label": "completed_group_4", "mean": 0.13539987802505493, "median": 0.13422535359859467, "p05": 0.12956365942955017, "p95": 0.14296601712703705 }, { "label": "completed_group_5", "mean": 0.1283380389213562, "median": 0.12796106934547424, "p05": 0.12316946685314178, "p95": 0.13433398306369781 }, { "label": "current_group_partial", "mean": 0.1254161298274994, "median": 0.12523823976516724, "p05": 0.12156305462121964, "p95": 0.12899920344352722 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458861351013184, "group": 6, "latest_source_mass": 0.14452049136161804, "layer": 24, "max_source_mass": 0.14452049136161804, "mixer_index": 46, "normalized_entropy": 0.9999876592688483, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14264336228370667, "median": 0.1426892876625061, "p05": 0.14129234850406647, "p95": 0.14382319152355194 }, { "label": "completed_group_1", "mean": 0.1430070698261261, "median": 0.14298146963119507, "p05": 0.14245536923408508, "p95": 0.1435846984386444 }, { "label": "completed_group_2", "mean": 0.14314037561416626, "median": 0.14310884475708008, "p05": 0.14277565479278564, "p95": 0.14360389113426208 }, { "label": "completed_group_3", "mean": 0.14221979677677155, "median": 0.14216911792755127, "p05": 0.14183637499809265, "p95": 0.14285607635974884 }, { "label": "completed_group_4", "mean": 0.14132915437221527, "median": 0.1413242518901825, "p05": 0.140913724899292, "p95": 0.14174436032772064 }, { "label": "completed_group_5", "mean": 0.1431397795677185, "median": 0.14310240745544434, "p05": 0.14276552200317383, "p95": 0.14377698302268982 }, { "label": "current_group_partial", "mean": 0.14452049136161804, "median": 0.1444905698299408, "p05": 0.14409083127975464, "p95": 0.14507406949996948 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9415628910064697, "group": 6, "latest_source_mass": 0.1248510330915451, "layer": 24, "max_source_mass": 0.16342055797576904, "mixer_index": 47, "normalized_entropy": 0.9977659512948458, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14895881712436676, "median": 0.14996516704559326, "p05": 0.13763123750686646, "p95": 0.15872029960155487 }, { "label": "completed_group_1", "mean": 0.16342055797576904, "median": 0.1639186143875122, "p05": 0.15447522699832916, "p95": 0.17102839052677155 }, { "label": "completed_group_2", "mean": 0.15406624972820282, "median": 0.1540084183216095, "p05": 0.1514483094215393, "p95": 0.1571783721446991 }, { "label": "completed_group_3", "mean": 0.14293310046195984, "median": 0.14186778664588928, "p05": 0.13832569122314453, "p95": 0.1491597443819046 }, { "label": "completed_group_4", "mean": 0.1363803744316101, "median": 0.13557225465774536, "p05": 0.13139937818050385, "p95": 0.14291447401046753 }, { "label": "completed_group_5", "mean": 0.12938985228538513, "median": 0.12914851307868958, "p05": 0.12471500039100647, "p95": 0.13491052389144897 }, { "label": "current_group_partial", "mean": 0.1248510330915451, "median": 0.12450477480888367, "p05": 0.120968759059906, "p95": 0.12821169197559357 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458723068237305, "group": 7, "latest_source_mass": 0.14437910914421082, "layer": 25, "max_source_mass": 0.14437910914421082, "mixer_index": 48, "normalized_entropy": 0.999980552939918, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14191991090774536, "median": 0.14168427884578705, "p05": 0.14056351780891418, "p95": 0.14397425949573517 }, { "label": "completed_group_1", "mean": 0.1418565809726715, "median": 0.1417136788368225, "p05": 0.14059004187583923, "p95": 0.14328429102897644 }, { "label": "completed_group_2", "mean": 0.14164230227470398, "median": 0.14142388105392456, "p05": 0.14061963558197021, "p95": 0.14320334792137146 }, { "label": "completed_group_3", "mean": 0.14289972186088562, "median": 0.14285793900489807, "p05": 0.1423202008008957, "p95": 0.14347271621227264 }, { "label": "completed_group_4", "mean": 0.14397987723350525, "median": 0.1439221203327179, "p05": 0.14329849183559418, "p95": 0.14462903141975403 }, { "label": "completed_group_5", "mean": 0.14332249760627747, "median": 0.14335353672504425, "p05": 0.14246267080307007, "p95": 0.14414797723293304 }, { "label": "completed_group_6", "mean": 0.14437910914421082, "median": 0.14459103345870972, "p05": 0.14338219165802002, "p95": 0.14524540305137634 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.073223114013672, "group": 7, "latest_source_mass": 0.11086538434028625, "layer": 25, "max_source_mass": 0.14827430248260498, "mixer_index": 49, "normalized_entropy": 0.9970095684146328, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13280925154685974, "median": 0.13328123092651367, "p05": 0.11957645416259766, "p95": 0.1447097212076187 }, { "label": "completed_group_1", "mean": 0.14827430248260498, "median": 0.14952056109905243, "p05": 0.13636183738708496, "p95": 0.15699218213558197 }, { "label": "completed_group_2", "mean": 0.13796664774417877, "median": 0.1381954699754715, "p05": 0.13450786471366882, "p95": 0.1413417011499405 }, { "label": "completed_group_3", "mean": 0.12756124138832092, "median": 0.12679177522659302, "p05": 0.12232367694377899, "p95": 0.13387270271778107 }, { "label": "completed_group_4", "mean": 0.12019774317741394, "median": 0.11888951063156128, "p05": 0.11402077227830887, "p95": 0.12811852991580963 }, { "label": "completed_group_5", "mean": 0.11310221254825592, "median": 0.11254298686981201, "p05": 0.10767743736505508, "p95": 0.12006338685750961 }, { "label": "completed_group_6", "mean": 0.10922321677207947, "median": 0.10910944640636444, "p05": 0.10472994297742844, "p95": 0.11491495370864868 }, { "label": "current_group_partial", "mean": 0.11086538434028625, "median": 0.11154524236917496, "p05": 0.10694204270839691, "p95": 0.11396989226341248 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079413890838623, "group": 7, "latest_source_mass": 0.1263430416584015, "layer": 26, "max_source_mass": 0.1263430416584015, "mixer_index": 50, "normalized_entropy": 0.9999867027561686, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12475188821554184, "median": 0.12463723123073578, "p05": 0.12364111095666885, "p95": 0.1259622424840927 }, { "label": "completed_group_1", "mean": 0.12460128962993622, "median": 0.12450964748859406, "p05": 0.12383783608675003, "p95": 0.125508114695549 }, { "label": "completed_group_2", "mean": 0.12424702197313309, "median": 0.12409250438213348, "p05": 0.1235705241560936, "p95": 0.12524327635765076 }, { "label": "completed_group_3", "mean": 0.12420950829982758, "median": 0.12421596050262451, "p05": 0.12374235689640045, "p95": 0.12457412481307983 }, { "label": "completed_group_4", "mean": 0.12484367191791534, "median": 0.12490126490592957, "p05": 0.12441762536764145, "p95": 0.1252550482749939 }, { "label": "completed_group_5", "mean": 0.12469653785228729, "median": 0.12477756291627884, "p05": 0.12382431328296661, "p95": 0.12524525821208954 }, { "label": "completed_group_6", "mean": 0.12630701065063477, "median": 0.12644168734550476, "p05": 0.12546542286872864, "p95": 0.12699897587299347 }, { "label": "current_group_partial", "mean": 0.1263430416584015, "median": 0.12645362317562103, "p05": 0.12567421793937683, "p95": 0.1267925351858139 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.072925567626953, "group": 7, "latest_source_mass": 0.10599855333566666, "layer": 26, "max_source_mass": 0.14843900501728058, "mixer_index": 51, "normalized_entropy": 0.9968664788491151, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13228744268417358, "median": 0.13281425833702087, "p05": 0.12057555466890335, "p95": 0.14273115992546082 }, { "label": "completed_group_1", "mean": 0.14843900501728058, "median": 0.14951951801776886, "p05": 0.13698264956474304, "p95": 0.1572267711162567 }, { "label": "completed_group_2", "mean": 0.13848665356636047, "median": 0.1387266218662262, "p05": 0.13503478467464447, "p95": 0.1420348733663559 }, { "label": "completed_group_3", "mean": 0.12843601405620575, "median": 0.12770286202430725, "p05": 0.12394566833972931, "p95": 0.13404949009418488 }, { "label": "completed_group_4", "mean": 0.12169650197029114, "median": 0.12072620540857315, "p05": 0.11642728000879288, "p95": 0.12845680117607117 }, { "label": "completed_group_5", "mean": 0.11442162096500397, "median": 0.11409753561019897, "p05": 0.10971812158823013, "p95": 0.12062239646911621 }, { "label": "completed_group_6", "mean": 0.11023421585559845, "median": 0.11024647206068039, "p05": 0.10620385408401489, "p95": 0.11548472940921783 }, { "label": "current_group_partial", "mean": 0.10599855333566666, "median": 0.10623474419116974, "p05": 0.1023266613483429, "p95": 0.10840564221143723 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0793263912200928, "group": 7, "latest_source_mass": 0.12782885134220123, "layer": 27, "max_source_mass": 0.12782885134220123, "mixer_index": 52, "normalized_entropy": 0.9999446243342576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12420415878295898, "median": 0.12419730424880981, "p05": 0.12226121127605438, "p95": 0.12590435147285461 }, { "label": "completed_group_1", "mean": 0.12273682653903961, "median": 0.12260904908180237, "p05": 0.12171939015388489, "p95": 0.12380002439022064 }, { "label": "completed_group_2", "mean": 0.12360995262861252, "median": 0.12356993556022644, "p05": 0.12293750047683716, "p95": 0.12423694133758545 }, { "label": "completed_group_3", "mean": 0.1238352432847023, "median": 0.12381016463041306, "p05": 0.12339816987514496, "p95": 0.12445184588432312 }, { "label": "completed_group_4", "mean": 0.1241140365600586, "median": 0.12404628843069077, "p05": 0.12358136475086212, "p95": 0.1247347816824913 }, { "label": "completed_group_5", "mean": 0.12607695162296295, "median": 0.1261017620563507, "p05": 0.12524889409542084, "p95": 0.12690351903438568 }, { "label": "completed_group_6", "mean": 0.127593994140625, "median": 0.12773489952087402, "p05": 0.1267755776643753, "p95": 0.1284082531929016 }, { "label": "current_group_partial", "mean": 0.12782885134220123, "median": 0.1278689205646515, "p05": 0.12723052501678467, "p95": 0.12838256359100342 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.073427200317383, "group": 7, "latest_source_mass": 0.10513068735599518, "layer": 27, "max_source_mass": 0.1465209424495697, "mixer_index": 53, "normalized_entropy": 0.9971077131807252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13135746121406555, "median": 0.13279247283935547, "p05": 0.11876361072063446, "p95": 0.14146237075328827 }, { "label": "completed_group_1", "mean": 0.1465209424495697, "median": 0.1469956636428833, "p05": 0.13593894243240356, "p95": 0.1559162735939026 }, { "label": "completed_group_2", "mean": 0.13772746920585632, "median": 0.137643963098526, "p05": 0.13452906906604767, "p95": 0.14171628654003143 }, { "label": "completed_group_3", "mean": 0.12966199219226837, "median": 0.1287793666124344, "p05": 0.12537093460559845, "p95": 0.13508695363998413 }, { "label": "completed_group_4", "mean": 0.12215200066566467, "median": 0.12143252789974213, "p05": 0.11671329289674759, "p95": 0.129044771194458 }, { "label": "completed_group_5", "mean": 0.11654195189476013, "median": 0.11616532504558563, "p05": 0.11162350326776505, "p95": 0.12263412773609161 }, { "label": "completed_group_6", "mean": 0.11090746521949768, "median": 0.11034701764583588, "p05": 0.10647446662187576, "p95": 0.11587852239608765 }, { "label": "current_group_partial", "mean": 0.10513068735599518, "median": 0.10488355159759521, "p05": 0.10140107572078705, "p95": 0.10828588157892227 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079422950744629, "group": 7, "latest_source_mass": 0.12521177530288696, "layer": 28, "max_source_mass": 0.12591606378555298, "mixer_index": 54, "normalized_entropy": 0.9999910596499906, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12477636337280273, "median": 0.12485313415527344, "p05": 0.12292245030403137, "p95": 0.12664391100406647 }, { "label": "completed_group_1", "mean": 0.12493627518415451, "median": 0.12481105327606201, "p05": 0.12430030107498169, "p95": 0.12584592401981354 }, { "label": "completed_group_2", "mean": 0.1242394968867302, "median": 0.12431945651769638, "p05": 0.12357322871685028, "p95": 0.12491215020418167 }, { "label": "completed_group_3", "mean": 0.12470392882823944, "median": 0.12473497539758682, "p05": 0.12409180402755737, "p95": 0.12546151876449585 }, { "label": "completed_group_4", "mean": 0.12560302019119263, "median": 0.12554584443569183, "p05": 0.12500521540641785, "p95": 0.12630382180213928 }, { "label": "completed_group_5", "mean": 0.12461308389902115, "median": 0.12459056079387665, "p05": 0.12410678714513779, "p95": 0.12512892484664917 }, { "label": "completed_group_6", "mean": 0.12591606378555298, "median": 0.12592178583145142, "p05": 0.12557773292064667, "p95": 0.12638571858406067 }, { "label": "current_group_partial", "mean": 0.12521177530288696, "median": 0.12524773180484772, "p05": 0.12447575479745865, "p95": 0.1259688436985016 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0725789070129395, "group": 7, "latest_source_mass": 0.10425455868244171, "layer": 28, "max_source_mass": 0.14827941358089447, "mixer_index": 55, "normalized_entropy": 0.9966997703328787, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13186156749725342, "median": 0.13315711915493011, "p05": 0.11918869614601135, "p95": 0.14256390929222107 }, { "label": "completed_group_1", "mean": 0.14827941358089447, "median": 0.14931634068489075, "p05": 0.13603198528289795, "p95": 0.15790212154388428 }, { "label": "completed_group_2", "mean": 0.13866659998893738, "median": 0.13880926370620728, "p05": 0.1348804384469986, "p95": 0.14264559745788574 }, { "label": "completed_group_3", "mean": 0.12988534569740295, "median": 0.12922212481498718, "p05": 0.12565752863883972, "p95": 0.13522912561893463 }, { "label": "completed_group_4", "mean": 0.12183371186256409, "median": 0.12081405520439148, "p05": 0.1163119301199913, "p95": 0.12904007732868195 }, { "label": "completed_group_5", "mean": 0.11524046957492828, "median": 0.1147729679942131, "p05": 0.11042807996273041, "p95": 0.12173305451869965 }, { "label": "completed_group_6", "mean": 0.1099783331155777, "median": 0.10993096977472305, "p05": 0.10556004196405411, "p95": 0.11569196730852127 }, { "label": "current_group_partial", "mean": 0.10425455868244171, "median": 0.10401678085327148, "p05": 0.10061369091272354, "p95": 0.10750719904899597 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0793380737304688, "group": 8, "latest_source_mass": 0.12683480978012085, "layer": 29, "max_source_mass": 0.12747105956077576, "mixer_index": 56, "normalized_entropy": 0.9999502424341857, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12417487800121307, "median": 0.1240256130695343, "p05": 0.12294038385152817, "p95": 0.1256686896085739 }, { "label": "completed_group_1", "mean": 0.1225123256444931, "median": 0.12230424582958221, "p05": 0.12168098986148834, "p95": 0.12367479503154755 }, { "label": "completed_group_2", "mean": 0.12277349829673767, "median": 0.1226874440908432, "p05": 0.12208077311515808, "p95": 0.12351377308368683 }, { "label": "completed_group_3", "mean": 0.1243998259305954, "median": 0.12447299808263779, "p05": 0.12367013841867447, "p95": 0.12505075335502625 }, { "label": "completed_group_4", "mean": 0.12567630410194397, "median": 0.12574473023414612, "p05": 0.12491010874509811, "p95": 0.1263245940208435 }, { "label": "completed_group_5", "mean": 0.12615728378295898, "median": 0.12627410888671875, "p05": 0.1253039538860321, "p95": 0.1267564296722412 }, { "label": "completed_group_6", "mean": 0.12747105956077576, "median": 0.12754422426223755, "p05": 0.1268254816532135, "p95": 0.1280263066291809 }, { "label": "completed_group_7", "mean": 0.12683480978012085, "median": 0.12676092982292175, "p05": 0.1260336935520172, "p95": 0.12750548124313354 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1906280517578125, "group": 8, "latest_source_mass": 0.09856453537940979, "layer": 29, "max_source_mass": 0.13291704654693604, "mixer_index": 57, "normalized_entropy": 0.9969977918295433, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11854428797960281, "median": 0.11933885514736176, "p05": 0.10912582278251648, "p95": 0.12757083773612976 }, { "label": "completed_group_1", "mean": 0.13291704654693604, "median": 0.13307878375053406, "p05": 0.12339349836111069, "p95": 0.14226418733596802 }, { "label": "completed_group_2", "mean": 0.12378417700529099, "median": 0.1235300600528717, "p05": 0.12091188877820969, "p95": 0.1279762238264084 }, { "label": "completed_group_3", "mean": 0.11745279282331467, "median": 0.11693152785301208, "p05": 0.11398620158433914, "p95": 0.12128151953220367 }, { "label": "completed_group_4", "mean": 0.11140252649784088, "median": 0.11079759150743484, "p05": 0.10681547224521637, "p95": 0.11677644401788712 }, { "label": "completed_group_5", "mean": 0.10409101843833923, "median": 0.1040785163640976, "p05": 0.09952308982610703, "p95": 0.10878291726112366 }, { "label": "completed_group_6", "mean": 0.09838292002677917, "median": 0.09840121120214462, "p05": 0.09399936348199844, "p95": 0.10255295783281326 }, { "label": "completed_group_7", "mean": 0.09486069530248642, "median": 0.09467358142137527, "p05": 0.09144656360149384, "p95": 0.09743223339319229 }, { "label": "current_group_partial", "mean": 0.09856453537940979, "median": 0.09877555072307587, "p05": 0.09461960196495056, "p95": 0.10164139419794083 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.197185516357422, "group": 8, "latest_source_mass": 0.11096570640802383, "layer": 30, "max_source_mass": 0.11249847710132599, "mixer_index": 58, "normalized_entropy": 0.999982222582434, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11084984987974167, "median": 0.11077362298965454, "p05": 0.108895443379879, "p95": 0.11311701685190201 }, { "label": "completed_group_1", "mean": 0.11021924018859863, "median": 0.11018644273281097, "p05": 0.10941747575998306, "p95": 0.11104057729244232 }, { "label": "completed_group_2", "mean": 0.11018813401460648, "median": 0.11026144027709961, "p05": 0.10942888259887695, "p95": 0.1108226552605629 }, { "label": "completed_group_3", "mean": 0.11046157777309418, "median": 0.11042240262031555, "p05": 0.10973355174064636, "p95": 0.11123604327440262 }, { "label": "completed_group_4", "mean": 0.11131839454174042, "median": 0.11132219433784485, "p05": 0.1105581521987915, "p95": 0.11199404299259186 }, { "label": "completed_group_5", "mean": 0.11167056858539581, "median": 0.1118055060505867, "p05": 0.11075222492218018, "p95": 0.1122528463602066 }, { "label": "completed_group_6", "mean": 0.11249847710132599, "median": 0.11257164180278778, "p05": 0.11187897622585297, "p95": 0.11291356384754181 }, { "label": "completed_group_7", "mean": 0.1118280440568924, "median": 0.11168843507766724, "p05": 0.11146453768014908, "p95": 0.11255130171775818 }, { "label": "current_group_partial", "mean": 0.11096570640802383, "median": 0.11086957156658173, "p05": 0.11044884473085403, "p95": 0.11181740462779999 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1899101734161377, "group": 8, "latest_source_mass": 0.09411308169364929, "layer": 30, "max_source_mass": 0.13352498412132263, "mixer_index": 59, "normalized_entropy": 0.9966710713162742, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1190595030784607, "median": 0.11995811760425568, "p05": 0.10872812569141388, "p95": 0.1278010904788971 }, { "label": "completed_group_1", "mean": 0.13352498412132263, "median": 0.13314831256866455, "p05": 0.12346142530441284, "p95": 0.14397428929805756 }, { "label": "completed_group_2", "mean": 0.1239522397518158, "median": 0.12362635135650635, "p05": 0.12061451375484467, "p95": 0.12857040762901306 }, { "label": "completed_group_3", "mean": 0.11779220402240753, "median": 0.11750936508178711, "p05": 0.11429227143526077, "p95": 0.12138796597719193 }, { "label": "completed_group_4", "mean": 0.11234382539987564, "median": 0.11192750930786133, "p05": 0.1077679693698883, "p95": 0.11798053979873657 }, { "label": "completed_group_5", "mean": 0.10513705015182495, "median": 0.1050291359424591, "p05": 0.10075211524963379, "p95": 0.11021918058395386 }, { "label": "completed_group_6", "mean": 0.09944089502096176, "median": 0.09912091493606567, "p05": 0.09499282389879227, "p95": 0.10404862463474274 }, { "label": "completed_group_7", "mean": 0.09463620185852051, "median": 0.09429345279932022, "p05": 0.090972401201725, "p95": 0.0975509062409401 }, { "label": "current_group_partial", "mean": 0.09411308169364929, "median": 0.09450676292181015, "p05": 0.09012747555971146, "p95": 0.0978383868932724 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.197190284729004, "group": 8, "latest_source_mass": 0.11225657165050507, "layer": 31, "max_source_mass": 0.11242865771055222, "mixer_index": 60, "normalized_entropy": 0.9999843927618645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11067462712526321, "median": 0.11063650250434875, "p05": 0.10940495133399963, "p95": 0.11191637068986893 }, { "label": "completed_group_1", "mean": 0.11021950840950012, "median": 0.11016013473272324, "p05": 0.10925821959972382, "p95": 0.11120235174894333 }, { "label": "completed_group_2", "mean": 0.11038416624069214, "median": 0.1102733239531517, "p05": 0.10960154980421066, "p95": 0.11125709116458893 }, { "label": "completed_group_3", "mean": 0.11103622615337372, "median": 0.11098648607730865, "p05": 0.11065587401390076, "p95": 0.11149194091558456 }, { "label": "completed_group_4", "mean": 0.11016052961349487, "median": 0.11016929149627686, "p05": 0.10976231843233109, "p95": 0.11053308844566345 }, { "label": "completed_group_5", "mean": 0.11144963651895523, "median": 0.11151091754436493, "p05": 0.11074185371398926, "p95": 0.11193440854549408 }, { "label": "completed_group_6", "mean": 0.11242865771055222, "median": 0.11247988045215607, "p05": 0.11164618283510208, "p95": 0.11297892034053802 }, { "label": "completed_group_7", "mean": 0.11139006912708282, "median": 0.11143708229064941, "p05": 0.11081651598215103, "p95": 0.1119559034705162 }, { "label": "current_group_partial", "mean": 0.11225657165050507, "median": 0.11227837204933167, "p05": 0.1117941215634346, "p95": 0.11280770599842072 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1896841526031494, "group": 8, "latest_source_mass": 0.09325435757637024, "layer": 31, "max_source_mass": 0.13428261876106262, "mixer_index": 61, "normalized_entropy": 0.9965682048112662, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11968795955181122, "median": 0.1202392578125, "p05": 0.11006385833024979, "p95": 0.12905627489089966 }, { "label": "completed_group_1", "mean": 0.13428261876106262, "median": 0.13435032963752747, "p05": 0.12363448739051819, "p95": 0.14427264034748077 }, { "label": "completed_group_2", "mean": 0.12373493611812592, "median": 0.12346962094306946, "p05": 0.12061072885990143, "p95": 0.1282179057598114 }, { "label": "completed_group_3", "mean": 0.1179073229432106, "median": 0.11775766313076019, "p05": 0.11464057117700577, "p95": 0.12117566168308258 }, { "label": "completed_group_4", "mean": 0.11132999509572983, "median": 0.11077313870191574, "p05": 0.10667262226343155, "p95": 0.1171092763543129 }, { "label": "completed_group_5", "mean": 0.10504785180091858, "median": 0.1049019992351532, "p05": 0.10071787983179092, "p95": 0.10987957566976547 }, { "label": "completed_group_6", "mean": 0.09994535893201828, "median": 0.09992853552103043, "p05": 0.09573008865118027, "p95": 0.1044701635837555 }, { "label": "completed_group_7", "mean": 0.09480959922075272, "median": 0.09453058242797852, "p05": 0.09107209742069244, "p95": 0.09781570732593536 }, { "label": "current_group_partial", "mean": 0.09325435757637024, "median": 0.09382589161396027, "p05": 0.08936256170272827, "p95": 0.09653929620981216 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1971938610076904, "group": 8, "latest_source_mass": 0.10965746641159058, "layer": 32, "max_source_mass": 0.11217275261878967, "mixer_index": 62, "normalized_entropy": 0.9999860203964374, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11116788536310196, "median": 0.11119179427623749, "p05": 0.11027558147907257, "p95": 0.11194087564945221 }, { "label": "completed_group_1", "mean": 0.1117495447397232, "median": 0.11171998083591461, "p05": 0.11123906075954437, "p95": 0.1124686524271965 }, { "label": "completed_group_2", "mean": 0.11143069714307785, "median": 0.11137667298316956, "p05": 0.11108753085136414, "p95": 0.1119472086429596 }, { "label": "completed_group_3", "mean": 0.11117202788591385, "median": 0.11118572950363159, "p05": 0.11080538481473923, "p95": 0.11158231645822525 }, { "label": "completed_group_4", "mean": 0.11217275261878967, "median": 0.11215600371360779, "p05": 0.1119328960776329, "p95": 0.11246690899133682 }, { "label": "completed_group_5", "mean": 0.11135637760162354, "median": 0.11132287979125977, "p05": 0.11110222339630127, "p95": 0.1116970106959343 }, { "label": "completed_group_6", "mean": 0.11160033196210861, "median": 0.11164811253547668, "p05": 0.11120022088289261, "p95": 0.111964151263237 }, { "label": "completed_group_7", "mean": 0.10969290882349014, "median": 0.10973770171403885, "p05": 0.10930771380662918, "p95": 0.10996787250041962 }, { "label": "current_group_partial", "mean": 0.10965746641159058, "median": 0.10969755053520203, "p05": 0.10922328382730484, "p95": 0.1099214255809784 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.188610792160034, "group": 8, "latest_source_mass": 0.09157620370388031, "layer": 32, "max_source_mass": 0.1360917091369629, "mixer_index": 63, "normalized_entropy": 0.9960796974214496, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11987313628196716, "median": 0.12083137035369873, "p05": 0.10883265733718872, "p95": 0.12953230738639832 }, { "label": "completed_group_1", "mean": 0.1360917091369629, "median": 0.13644108176231384, "p05": 0.12369051575660706, "p95": 0.14713600277900696 }, { "label": "completed_group_2", "mean": 0.12462760508060455, "median": 0.12453563511371613, "p05": 0.12107435613870621, "p95": 0.12926824390888214 }, { "label": "completed_group_3", "mean": 0.11755925416946411, "median": 0.11717041581869125, "p05": 0.11395573616027832, "p95": 0.12139421701431274 }, { "label": "completed_group_4", "mean": 0.11237476021051407, "median": 0.11172428727149963, "p05": 0.10730767995119095, "p95": 0.11879656463861465 }, { "label": "completed_group_5", "mean": 0.10438287258148193, "median": 0.10406170785427094, "p05": 0.09989674389362335, "p95": 0.10985209792852402 }, { "label": "completed_group_6", "mean": 0.09920359402894974, "median": 0.09911085665225983, "p05": 0.0946069061756134, "p95": 0.10452912747859955 }, { "label": "completed_group_7", "mean": 0.09431086480617523, "median": 0.09389802813529968, "p05": 0.09041668474674225, "p95": 0.09788760542869568 }, { "label": "current_group_partial", "mean": 0.09157620370388031, "median": 0.09197241812944412, "p05": 0.0875491350889206, "p95": 0.09489993005990982 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1930694580078125, "group": null, "latest_source_mass": 0.11946354806423187, "layer": null, "max_source_mass": 0.12094368040561676, "mixer_index": null, "normalized_entropy": 0.9981089236979843, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.10071488469839096, "median": 0.09996568411588669, "p05": 0.09106086939573288, "p95": 0.1130409762263298 }, { "label": "completed_group_1", "mean": 0.09411877393722534, "median": 0.09266924858093262, "p05": 0.08850140869617462, "p95": 0.10204548388719559 }, { "label": "completed_group_2", "mean": 0.10181896388530731, "median": 0.10170605033636093, "p05": 0.09935513883829117, "p95": 0.10529708862304688 }, { "label": "completed_group_3", "mean": 0.10989432036876678, "median": 0.11005622148513794, "p05": 0.1059446707367897, "p95": 0.11359670758247375 }, { "label": "completed_group_4", "mean": 0.1145806610584259, "median": 0.11529573798179626, "p05": 0.10914603620767593, "p95": 0.11935751885175705 }, { "label": "completed_group_5", "mean": 0.11861918121576309, "median": 0.11936421692371368, "p05": 0.11333125829696655, "p95": 0.12279194593429565 }, { "label": "completed_group_6", "mean": 0.11984598636627197, "median": 0.12033014744520187, "p05": 0.11644290387630463, "p95": 0.12263244390487671 }, { "label": "completed_group_7", "mean": 0.12094368040561676, "median": 0.12095880508422852, "p05": 0.118732750415802, "p95": 0.12314030528068542 }, { "label": "completed_group_8", "mean": 0.11946354806423187, "median": 0.118886798620224, "p05": 0.11734164506196976, "p95": 0.12209592759609222 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", "optimizer_state_before": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.002919268561527133, 0.0026904908008873463, 0.0028405722696334124, 0.0029144894797354937, 0.0030634324066340923, 0.0032732728868722916, 0.003848863998427987, 0.003392934100702405, 0.0037359180860221386, 0.004302032291889191, 0.004344160668551922, 0.004835603758692741, 0.005531937349587679, 0.005483300890773535, 0.005397035740315914, 0.005739198997616768, 0.005659298039972782, 0.007230010814964771, 0.006938732694834471, 0.007343437522649765, 0.007882392033934593, 0.00830139685422182, 0.008992243558168411, 0.009393857792019844, 0.012831038795411587, 0.00905293133109808, 0.012243949808180332, 0.012258118949830532, 0.013431218452751637, 0.01483316533267498, 0.015388047322630882, 0.014634119346737862 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.570841959619429e-05, "first_to_last_ratio": 3.472594525431938, "imbalance_abs_log_ratio": 1.2449020164225795, "last_quartile_mean": 4.523539814726973e-06, "mean": 8.683279858701098e-06, "normalized": [ 2.229894338065868, 2.1043034837063286, 1.976946278509285, 1.8612974320657731, 1.7187480804487953, 1.6201790212455152, 1.5254325865776683, 1.4355362848487128, 1.3017373037261373, 1.2100830304331005, 1.133220479031311, 1.0535486714736453, 0.9690659040803394, 0.9036010498699347, 0.8438110062145913, 0.7926660684312774, 0.749751881449241, 0.7082035160351038, 0.6752208628773162, 0.6475764877057508, 0.6244377002240272, 0.6007972561165787, 0.5813749906205735, 0.5649799471303525, 0.5550363681005585, 0.5437815898914894, 0.5335487688172357, 0.5248093961192007, 0.5123845047471113, 0.5051729354463279, 0.4992577444849533, 0.49359503150589623 ], "peak_layer": 1, "peak_normalized": 2.229894338065868, "population_cv": 0.5276859177486317, "rest_mean": 9.350082985259144e-06, "spike_contrast": 0.543582659458801, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.082542975287651e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.936279659275897e-05, 1.8272256056661718e-05, 1.7166377801913768e-05, 1.6162166502908804e-05, 1.49243705891422e-05, 1.4068467862671241e-05, 1.3245758054836188e-05, 1.2465163308661431e-05, 1.1303349310765043e-05, 1.050748960551573e-05, 9.840070561040193e-06, 9.148257959168404e-06, 8.414670446654782e-06, 7.846220796636771e-06, 7.3270471148134675e-06, 6.8829413066850975e-06, 6.510305411211448e-06, 6.149529326648917e-06, 5.863131718797376e-06, 5.623087872663746e-06, 5.42216730536893e-06, 5.216890713199973e-06, 5.048241746408166e-06, 4.905878995487001e-06, 4.819536115974188e-06, 4.7218077270372305e-06, 4.632953277905472e-06, 4.557066858978942e-06, 4.4491780499811284e-06, 4.38655797552201e-06, 4.335194716986734e-06, 4.286023795430083e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.106548153184121e-05, "first_to_last_ratio": 3.396664379322294, "imbalance_abs_log_ratio": 1.2227938851577476, "last_quartile_mean": 1.7978073401536676e-05, "mean": 3.421872185072061e-05, "normalized": [ 2.125690241222297, 2.035753111677548, 1.9424653444151043, 1.8443604241196392, 1.713002582679694, 1.6251047960941019, 1.5375111780105508, 1.4526202049778152, 1.3158411261187606, 1.226794175835664, 1.149073116741286, 1.0695301297827082, 0.9835511244361359, 0.9163244633971447, 0.8552224227935168, 0.8029396208596036, 0.7578907347648619, 0.7156154592837152, 0.6817576032295484, 0.6538669675872317, 0.6299332335154186, 0.605919603383038, 0.5863501185278558, 0.569785799336057, 0.5595323652477712, 0.5483698809054829, 0.5380511691309848, 0.5293393622233508, 0.5166837305455455, 0.5094821346031305, 0.5036034749610113, 0.498034299593426 ], "peak_layer": 1, "peak_normalized": 2.125690241222297, "population_cv": 0.5121606369233174, "rest_mean": 3.681488218447797e-05, "spike_contrast": 0.5486763735168884, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0199456048430876e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.273840310517699e-05, 6.966086948523298e-05, 6.646868132520467e-05, 6.311165634542704e-05, 5.8616758906282485e-05, 5.560900899581611e-05, 5.261166734271683e-05, 4.970680674887262e-05, 4.502640149439685e-05, 4.197932867100462e-05, 3.931981336791068e-05, 3.659795402199961e-05, 3.365586235304363e-05, 3.1355451937997714e-05, 2.9264618206070736e-05, 2.747556754911784e-05, 2.593405224615708e-05, 2.448744635330513e-05, 2.3328873794525862e-05, 2.2374491891241632e-05, 2.1555510102189146e-05, 2.073379437206313e-05, 2.006415161304176e-05, 1.9497341781971045e-05, 1.9146482372889295e-05, 1.8764516426017508e-05, 1.8411423297948204e-05, 1.8113316400558688e-05, 1.76802568603307e-05, 1.7433827451895922e-05, 1.723266723274719e-05, 1.70420971699059e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.054153479635715485, "first_to_last_ratio": 3.3966644557946295, "imbalance_abs_log_ratio": 1.2227939076716985, "last_quartile_mean": 0.01594313490204513, "mean": 0.030345504521392286, "normalized": [ 2.1256903113360446, 2.035753146214189, 1.9424653011478115, 1.8443605007189683, 1.71300252819031, 1.6251048166450035, 1.5375111878366567, 1.452620175683203, 1.3158411334809852, 1.2267941193182312, 1.1490731743165223, 1.0695301881432524, 0.9835511318647784, 0.9163244786384582, 0.8552223856485347, 0.8029396127492687, 0.7578907435068827, 0.7156154476500971, 0.6817576088078939, 0.6538669486644281, 0.6299332311205004, 0.6059195951826442, 0.586350084864764, 0.569785800788618, 0.5595323690576035, 0.5483698809409665, 0.5380511395546987, 0.5293393594226915, 0.5166837108984478, 0.5094821244722924, 0.5036034882338023, 0.4980342749014515 ], "peak_layer": 1, "peak_normalized": 2.1256903113360446, "population_cv": 0.5121606472523333, "rest_mean": 0.0326478054953946, "spike_contrast": 0.5486763655311122, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017913079261779784, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06450514495372772, 0.06177595630288124, 0.05894508957862854, 0.05596804991364479, 0.05198192596435547, 0.049314625561237335, 0.04665655270218849, 0.04408049210906029, 0.039929863065481186, 0.03722768649458885, 0.034869205206632614, 0.032455433160066605, 0.029846355319023132, 0.027806328609585762, 0.025952154770493507, 0.024365607649087906, 0.02299857698380947, 0.021715711802244186, 0.02068827860057354, 0.01984192244708538, 0.019115641713142395, 0.01838693581521511, 0.017793089151382446, 0.01729043759405613, 0.016979292035102844, 0.01664056070148945, 0.01632743328809738, 0.01606306992471218, 0.015679027885198593, 0.015460492111742496, 0.015282101929187775, 0.015113101340830326 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.484954840540013e-06, "first_to_last_ratio": 4.317118296735341, "imbalance_abs_log_ratio": 1.462588118730986, "last_quartile_mean": 1.2705129819323702e-06, "mean": 2.775344690064685e-06, "normalized": [ 2.642311887415898, 2.416230245995366, 2.21143682159549, 2.0333492354809493, 1.8430856012839139, 1.6939904160227164, 1.5446558421616008, 1.4254585070372157, 1.2827451334693212, 1.1755524566426843, 1.0923958043799127, 1.0016640021807137, 0.9226630606403423, 0.8576190807297013, 0.7936412075441582, 0.7370849544400642, 0.6937134258189417, 0.6483319177623661, 0.6136592809430983, 0.5846488972705621, 0.5617626261670844, 0.5391223387293338, 0.5197976899046192, 0.5027943403920482, 0.49023336166728243, 0.4787835081603097, 0.4696409445710295, 0.46085508801940983, 0.450168541694312, 0.44355065941556593, 0.43718887869853934, 0.4318642437654502 ], "peak_layer": 1, "peak_normalized": 2.642311887415898, "population_cv": 0.6359571845984164, "rest_mean": 3.020632700807053e-06, "spike_contrast": 0.48029322852403633, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.450789432055899e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.333326266234508e-06, 6.705871783196926e-06, 6.1374994402285665e-06, 5.643245003739139e-06, 5.1151978368579876e-06, 4.701407306129113e-06, 4.286952389520593e-06, 3.95613869841327e-06, 3.5600598948803963e-06, 3.2625632684357697e-06, 3.031774895134731e-06, 2.779962869681185e-06, 2.5607080260670045e-06, 2.380188561801333e-06, 2.202627911174204e-06, 2.0456648144318024e-06, 1.9252938727731816e-06, 1.7993445453612367e-06, 1.7031160268743406e-06, 1.622602212592028e-06, 1.5590849216096103e-06, 1.4962503200877109e-06, 1.4426177585846744e-06, 1.3954276028016466e-06, 1.3605665571958525e-06, 1.3287892670632573e-06, 1.3034155017521698e-06, 1.2790317214239622e-06, 1.2493728718254715e-06, 1.2310059673836804e-06, 1.2133498330513248e-06, 1.1985721357632428e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.000926375942072e-05, "first_to_last_ratio": 2.861076547442411, "imbalance_abs_log_ratio": 1.0511979691949909, "last_quartile_mean": 1.7479177131463075e-05, "mean": 3.001758557275025e-05, "normalized": [ 1.966517821002492, 1.8844494706669952, 1.8016591622079403, 1.7146543004927615, 1.602713342111759, 1.525842314618408, 1.4518503935852287, 1.3803041859872354, 1.2718526716709437, 1.1969100977048808, 1.1317671023781757, 1.06503487809827, 0.9973441232158428, 0.941363281859895, 0.8896494105618908, 0.8445592656875696, 0.8090600018046681, 0.7713325693634966, 0.7401533589124044, 0.7139326567185207, 0.692297727813069, 0.6685034690624881, 0.6485427267474059, 0.6313224439068734, 0.6206897714046083, 0.6084823527417841, 0.5970360381608709, 0.5872159169829965, 0.572597446631392, 0.564250570870669, 0.5573663965692749, 0.5507447304591896 ], "peak_layer": 1, "peak_normalized": 1.966517821002492, "population_cv": 0.43784911874358545, "rest_mean": 3.1950544494234406e-05, "spike_contrast": 0.6128098192589182, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.957960739673581e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.9030116972280666e-05, 5.656662324327044e-05, 5.408145807450637e-05, 5.146978219272569e-05, 4.810958489542827e-05, 4.580210224958137e-05, 4.3581043428275734e-05, 4.143339901929721e-05, 3.817794640781358e-05, 3.592835128074512e-05, 3.397291584406048e-05, 3.196977559127845e-05, 2.993786256411113e-05, 2.8257452868274413e-05, 2.670512731128838e-05, 2.5351630029035732e-05, 2.4286027837661095e-05, 2.3153541405918077e-05, 2.2217616788111627e-05, 2.1430534616229124e-05, 2.0781106286449358e-05, 2.006686008826364e-05, 1.9467686797725037e-05, 1.8950775483972393e-05, 1.8631608327268623e-05, 1.8265171092934906e-05, 1.7921580365509726e-05, 1.7626804037718102e-05, 1.7187992852996103e-05, 1.6937439795583487e-05, 1.673079350439366e-05, 1.6532027075299993e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.89006590921781e-05, "first_to_last_ratio": 2.1665975586085744, "imbalance_abs_log_ratio": 0.7731579916979808, "last_quartile_mean": 1.7954723034563358e-05, "mean": 2.587040432899812e-05, "normalized": [ 1.7831448102947671, 1.6974226005892297, 1.620036533641332, 1.533717800359919, 1.4465980773830347, 1.3745334732258696, 1.316859965180907, 1.257080202982369, 1.1876570648434144, 1.1203272435244702, 1.0718495078032313, 1.0193313865605265, 0.978296344153428, 0.937610047259807, 0.9066038289594328, 0.877320807240572, 0.8649781691504772, 0.8401000985755385, 0.8145599740177561, 0.792555397726043, 0.7795027531450014, 0.7587311609912605, 0.7416782788097599, 0.7272996278499536, 0.7222997702968516, 0.7135070943498951, 0.7058159198574021, 0.6988176661381593, 0.6867067750666979, 0.6814848031835403, 0.675082580859005, 0.668490235980348 ], "peak_layer": 1, "peak_normalized": 1.7831448102947671, "population_cv": 0.3297379118959793, "rest_mean": 2.708773947098827e-05, "spike_contrast": 0.7123811340152144, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9296794562251307e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.6130677219480276e-05, 4.391300899442285e-05, 4.191100015304983e-05, 3.967789962189272e-05, 3.742407716345042e-05, 3.555973671609536e-05, 3.406769974390045e-05, 3.252117312513292e-05, 3.072516847169027e-05, 2.8983318770769984e-05, 2.7729180146707222e-05, 2.63705151155591e-05, 2.530892197682988e-05, 2.4256351025542244e-05, 2.3454207621398382e-05, 2.269664400955662e-05, 2.2377334971679375e-05, 2.173372922698036e-05, 2.1072995878057554e-05, 2.050372859230265e-05, 2.0166051399428397e-05, 1.9628681911854073e-05, 1.9187516954843886e-05, 1.8815535440808162e-05, 1.8686187104322016e-05, 1.8458717022440396e-05, 1.8259743228554726e-05, 1.8078695575241e-05, 1.776538192643784e-05, 1.7630287402425893e-05, 1.7464659322286025e-05, 1.729411269479897e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011786452796513913, "first_to_last_ratio": 5.272419649365271, "imbalance_abs_log_ratio": 1.6624893936843081, "last_quartile_mean": 2.2354921611622558e-05, "mean": 5.981176059322024e-05, "normalized": [ 2.342046121512161, 2.2441163908348334, 2.142933559820905, 2.032574241487723, 1.8909432158278896, 1.7811012647720825, 1.7046454997674345, 1.6263691566595195, 1.4513535263674326, 1.3484633711734058, 1.2565665103260477, 1.1592512217173825, 1.0440274588303153, 0.9501757209203151, 0.8639054268917737, 0.7923577171085253, 0.7123005746652016, 0.6458221031841243, 0.5963034371094506, 0.5569297557861542, 0.5166112374905155, 0.4756772431215453, 0.44740588281370386, 0.42808243396944073, 0.41207290926584317, 0.3977052623291704, 0.38540407372369656, 0.3748332006962577, 0.3630559483389979, 0.3565727390637327, 0.35157439090519904, 0.34881840351921967 ], "peak_layer": 1, "peak_normalized": 2.342046121512161, "population_cv": 0.6458102290367006, "rest_mean": 6.583757459776718e-05, "spike_contrast": 0.41423708475420845, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7272364968666805e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014008190191816539, 0.00013422455231193453, 0.00012817262904718518, 0.00012157184391980991, 0.00011310064292047173, 0.00010653080244082958, 0.00010195784852840006, 9.727600263431668e-05, 8.680800965521485e-05, 8.065396832535043e-05, 7.515745528507978e-05, 6.933685654075816e-05, 6.244512042030692e-05, 5.683168274117634e-05, 5.167170456843451e-05, 4.739231007988565e-05, 4.2603951442288235e-05, 3.862775702145882e-05, 3.566595842130482e-05, 3.331094922032207e-05, 3.089942765654996e-05, 2.8451093385228887e-05, 2.6760133550851606e-05, 2.56043640547432e-05, 2.464680619596038e-05, 2.3787451937096193e-05, 2.3051696189213544e-05, 2.241943366243504e-05, 2.171501546399668e-05, 2.1327243302948773e-05, 2.1028283299528994e-05, 2.086344284180086e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.005079356953501701, 0.0052208867855370045, 0.005689430516213179, 0.005868426524102688, 0.00590060418471694, 0.0059102908708155155, 0.0058776624500751495, 0.006663382984697819, 0.006277414970099926, 0.006413500756025314, 0.006856496445834637, 0.006499431096017361, 0.006718362681567669, 0.0070298356004059315, 0.006969383917748928, 0.0068367342464625835, 0.007033169735223055, 0.007157569285482168, 0.0075136395171284676, 0.007708695251494646, 0.007887008599936962, 0.008205465972423553, 0.00814912747591734, 0.008287571370601654, 0.008026733994483948, 0.008120592683553696, 0.008479002863168716, 0.00907627958804369, 0.00841403380036354, 0.008773921057581902, 0.008821850642561913, 0.009164483286440372 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5008465311439068e-05, "first_to_last_ratio": 3.3668968034019833, "imbalance_abs_log_ratio": 1.213991490288974, "last_quartile_mean": 4.457655279566097e-06, "mean": 8.299250268350988e-06, "normalized": [ 2.2568151540450523, 2.1152935419859507, 1.9767562187382288, 1.86698957878165, 1.7143188557206628, 1.6128346254213421, 1.5135326124803135, 1.410756892623042, 1.2736883716277778, 1.1902867485409472, 1.1070374521817832, 1.0322313304507316, 0.9480382659054776, 0.8848529200942086, 0.8311995014573598, 0.7835740569083032, 0.742040914352895, 0.7071972712748903, 0.6782348645232779, 0.6514608668278806, 0.6287747596064291, 0.6086256047198574, 0.5913261103101772, 0.5772101349211559, 0.569144818247716, 0.5582752103529628, 0.549190400479895, 0.5409965375823161, 0.5285413070813697, 0.5224600186388942, 0.5164682084504763, 0.5118468456669756 ], "peak_layer": 1, "peak_normalized": 2.2568151540450523, "population_cv": 0.524209711679407, "rest_mean": 8.921669009227858e-06, "spike_contrast": 0.5535050742757021, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.9381890676158944e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8729873772826977e-05, 1.7555350495968014e-05, 1.640559457882773e-05, 1.5494613762712106e-05, 1.422756122337887e-05, 1.338531819783384e-05, 1.2561185940285213e-05, 1.1708224519679789e-05, 1.0570658560027368e-05, 9.878487617243081e-06, 9.187580872094259e-06, 8.566746146243531e-06, 7.86800683272304e-06, 7.343615834543016e-06, 6.898332685523201e-06, 6.5030772020691074e-06, 6.1583832575706765e-06, 5.8692071434052195e-06, 5.62884088139981e-06, 5.406636773841456e-06, 5.2183590923959855e-06, 5.051136213296559e-06, 4.907563379674684e-06, 4.790411367139313e-06, 4.723475285572931e-06, 4.633265689335531e-06, 4.557868578558555e-06, 4.489865659706993e-06, 4.38649658462964e-06, 4.336026449891506e-06, 4.286298917577369e-06, 4.247945071256254e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.820503929498955e-05, "first_to_last_ratio": 3.2848952430521337, "imbalance_abs_log_ratio": 1.1893347619462469, "last_quartile_mean": 1.7718994058668613e-05, "mean": 3.2650613661644456e-05, "normalized": [ 2.1450013507239705, 2.0461366500203595, 1.9398189683428497, 1.8522773613526717, 1.7094590145500226, 1.6168974254280155, 1.5255888819635426, 1.4261239538940118, 1.2879114356833505, 1.2060871976977126, 1.1221510774968064, 1.0481371490840103, 0.9613461884209538, 0.8968067603991513, 0.8421359861574166, 0.7933538356930859, 0.7503585255090798, 0.7148355279210235, 0.6855522988186872, 0.6585248527378349, 0.6351822812313331, 0.6146906504361358, 0.5972743074473386, 0.5828693520617939, 0.574802048908203, 0.5639397943233103, 0.554845965325609, 0.5465864044553121, 0.5340118249846835, 0.5278506086231465, 0.5219704528761507, 0.5174718674324269 ], "peak_layer": 1, "peak_normalized": 2.1450013507239705, "population_cv": 0.5072019698199706, "rest_mean": 3.5063350541922434e-05, "spike_contrast": 0.5596109386261622, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9621834508143365e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.003561040619388e-05, 6.680761725874618e-05, 6.333627970889211e-05, 6.047799251973629e-05, 5.581488585448824e-05, 5.279269316815771e-05, 4.981141319149174e-05, 4.656382225221023e-05, 4.205109871691093e-05, 3.937948713428341e-05, 3.663892130134627e-05, 3.422232111915946e-05, 3.138854299322702e-05, 2.9281291062943637e-05, 2.749625673459377e-05, 2.5903489586198702e-05, 2.4499666324118152e-05, 2.3339818653766997e-05, 2.238370325358119e-05, 2.1501240553334355e-05, 2.0739091269206256e-05, 2.007002694881521e-05, 1.9501372662489302e-05, 1.903104202938266e-05, 1.8767639630823396e-05, 1.841298035287764e-05, 1.811606125556864e-05, 1.7846381524577737e-05, 1.7435813788324594e-05, 1.7234646293218248e-05, 1.704265559965279e-05, 1.6895774024305865e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05161681119352579, "first_to_last_ratio": 3.284895287464527, "imbalance_abs_log_ratio": 1.1893347754664323, "last_quartile_mean": 0.015713380998931825, "mean": 0.028954890876775607, "normalized": [ 2.145001289725201, 2.046136691673054, 1.939818911776489, 1.8522774119626542, 1.7094589830823987, 1.6168973606656405, 1.5255889998464391, 1.4261239529824867, 1.287911366616444, 1.2060871893436458, 1.1221510842273104, 1.0481372445524282, 0.961346209259541, 0.8968067750917011, 0.8421359558072424, 0.7933538963986676, 0.7503585064155442, 0.7148355327095294, 0.6855522990290889, 0.6585248332975209, 0.6351822731624658, 0.6146906547931704, 0.5972742941320782, 0.5828693766065183, 0.5748020438293477, 0.5639398000251236, 0.5548459661026797, 0.5465864176570202, 0.534011826874325, 0.527850568480463, 0.5219704287182262, 0.5174718551555549 ], "peak_layer": 1, "peak_normalized": 2.145001289725201, "population_cv": 0.5072019680935582, "rest_mean": 0.03109453007992771, "spike_contrast": 0.5596109391274233, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017400839179754258, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06210827827453613, 0.05924566462635994, 0.05616724491119385, 0.05363249033689499, 0.049497198313474655, 0.04681708663702011, 0.044173263013362885, 0.04129326343536377, 0.037291333079338074, 0.034922122955322266, 0.032491762191057205, 0.030348699539899826, 0.027835674583911896, 0.025966942310333252, 0.02438395470380783, 0.02297147549688816, 0.021726548671722412, 0.020697984844446182, 0.019850092008709908, 0.019067514687776566, 0.01839163340628147, 0.017798300832509995, 0.017294012010097504, 0.01687691919505596, 0.016643330454826355, 0.01632881537079811, 0.016065504401922226, 0.015826350077986717, 0.015462254174053669, 0.015283855609595776, 0.015113596804440022, 0.014983341097831726 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.355004418561293e-06, "first_to_last_ratio": 4.2803271552883535, "imbalance_abs_log_ratio": 1.4540294448239433, "last_quartile_mean": 1.251073626917787e-06, "mean": 2.683941971781678e-06, "normalized": [ 2.7030485685097614, 2.4621902558382964, 2.2386850638453395, 2.05248673466923, 1.8575399210018237, 1.6957758104696088, 1.5437340831941542, 1.4081496862748333, 1.2505669189373825, 1.154757255828014, 1.0675003825206393, 0.9762526930125442, 0.897424076523062, 0.8318154411009656, 0.7746171026378299, 0.7211559397261628, 0.6781980327113627, 0.6399672113971655, 0.60846523760761, 0.5811474026512373, 0.5581318010721549, 0.5402081583654614, 0.5217772343602641, 0.5073418706694675, 0.49649585255921586, 0.4853123246471542, 0.4772675785905054, 0.4696153655827239, 0.458545908189773, 0.453049428909843, 0.4466290334907211, 0.4421476251056929 ], "peak_layer": 1, "peak_normalized": 2.7030485685097614, "population_cv": 0.6457571562979046, "rest_mean": 2.9201333467830814e-06, "spike_contrast": 0.48234391361810997, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4085085467741009e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.254825504787732e-06, 6.608375770156272e-06, 6.008500804455252e-06, 5.508755293703871e-06, 4.985529358236818e-06, 4.551363872451475e-06, 4.143292699154699e-06, 3.7793920455442276e-06, 3.3564490422577364e-06, 3.0993014661362395e-06, 2.8651090815401403e-06, 2.6202055778412614e-06, 2.4086341454676585e-06, 2.232544375146972e-06, 2.079027353829588e-06, 1.9355406948307063e-06, 1.82024416517379e-06, 1.7176348592329305e-06, 1.633085389585176e-06, 1.5597659057675628e-06, 1.4979933666836587e-06, 1.4498873497359455e-06, 1.4004198192196782e-06, 1.3616761407320155e-06, 1.332566057499207e-06, 1.3025501175434329e-06, 1.280958485949668e-06, 1.2604203902810696e-06, 1.2307106089792796e-06, 1.2159583775428473e-06, 1.198726408802031e-06, 1.1866985687447595e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.78605675198196e-05, "first_to_last_ratio": 2.7793570801926766, "imbalance_abs_log_ratio": 1.0222196348386712, "last_quartile_mean": 1.7220013887708774e-05, "mean": 2.881263026210945e-05, "normalized": [ 1.9752570579468534, 1.888104295535499, 1.7945647548145947, 1.7154982752112686, 1.5967511433618535, 1.5186186336526764, 1.441625524284827, 1.3583546018661343, 1.2494560102010674, 1.1811084039550475, 1.1110423537101946, 1.0496234480105349, 0.9817226877095458, 0.9275068527847158, 0.8805127756083984, 0.8381970075463177, 0.803976865779107, 0.7714567335637176, 0.7440882717080923, 0.7183088339322109, 0.696644002608348, 0.6758702747761651, 0.6578620296540808, 0.6426087873852322, 0.6340400523715064, 0.6220565885755293, 0.611876165213302, 0.6024355818665901, 0.5879093007530156, 0.5807453084805843, 0.5738005724905904, 0.5683768046423998 ], "peak_layer": 1, "peak_normalized": 1.9752570579468534, "population_cv": 0.4317079414337261, "rest_mean": 3.0619262057977416e-05, "spike_contrast": 0.622380073312624, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.905681856442243e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.691235128324479e-05, 5.440125096356496e-05, 5.1706130761886016e-05, 4.9428017518948764e-05, 4.6006600314285606e-05, 4.375539720058441e-05, 4.153702320763841e-05, 3.9137768908403814e-05, 3.600011405069381e-05, 3.4030839742626995e-05, 3.201205254299566e-05, 3.0242412321968004e-05, 2.8286012820899487e-05, 2.6723912014858797e-05, 2.5369889044668525e-05, 2.4150660465238616e-05, 2.3164688172983006e-05, 2.2227697627386078e-05, 2.14391402550973e-05, 2.069636684609577e-05, 2.0072146071470343e-05, 1.9473600332275964e-05, 1.8954735423903912e-05, 1.8515249394113198e-05, 1.8268361600348726e-05, 1.7923086488735862e-05, 1.7629761714488268e-05, 1.7357753677060828e-05, 1.6939213310251944e-05, 1.673279984970577e-05, 1.6532703739358112e-05, 1.637643072172068e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.7207858895271784e-05, "first_to_last_ratio": 2.0918563583707814, "imbalance_abs_log_ratio": 0.7380518814986469, "last_quartile_mean": 1.778700470822514e-05, "mean": 2.5014859602379147e-05, "normalized": [ 1.766896017685199, 1.6852067556838466, 1.5917675487924012, 1.5166567899677899, 1.4315811273281913, 1.3662079609937838, 1.303579546183775, 1.2375462607814267, 1.160212708426433, 1.1098915366489155, 1.0567965268051207, 1.0112629998208633, 0.9697402160962814, 0.9378324250878841, 0.907941204027948, 0.8814449641361121, 0.8690589692914835, 0.8427704073086626, 0.8201725639084954, 0.8014260695185076, 0.784571291095966, 0.7669834591017225, 0.7521317253856684, 0.7398605496651066, 0.7380371143547211, 0.7300178093260107, 0.722929928084616, 0.7169054508143122, 0.7047914878526057, 0.6980364293666598, 0.6913059490086015, 0.6864362074508885 ], "peak_layer": 1, "peak_normalized": 1.766896017685199, "population_cv": 0.3175074310788083, "rest_mean": 2.614369299812501e-05, "spike_contrast": 0.7236605504321191, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8919159265351483e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.419865581439808e-05, 4.215521039441228e-05, 3.981784175266512e-05, 3.79389566660393e-05, 3.581080090953037e-05, 3.417550033191219e-05, 3.260885932832025e-05, 3.095704596489668e-05, 2.9022558010183275e-05, 2.776378096314147e-05, 2.6435616746312007e-05, 2.5296601961599663e-05, 2.4257915356429294e-05, 2.345974644413218e-05, 2.2712021745974198e-05, 2.2049222025088966e-05, 2.173938810301479e-05, 2.1081783415866084e-05, 2.051650153589435e-05, 2.0047560610692017e-05, 1.962594069482293e-05, 1.9185983546776697e-05, 1.8814469513017684e-05, 1.8507507775211707e-05, 1.846189479692839e-05, 1.8261293007526547e-05, 1.808399065339472e-05, 1.793328920030035e-05, 1.763026011758484e-05, 1.7461283277953044e-05, 1.7292921256739646e-05, 1.7171105355373584e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011250984425714705, "first_to_last_ratio": 5.136972637567696, "imbalance_abs_log_ratio": 1.6364639249316602, "last_quartile_mean": 2.1901974605498253e-05, "mean": 5.6501408323583746e-05, "normalized": [ 2.382164866319327, 2.279581056980086, 2.1640865436402454, 2.0688186882965165, 1.8923495642236086, 1.8092596842626791, 1.7252927191320069, 1.6086472166206554, 1.4275667956980447, 1.3299589734449933, 1.2272752348601612, 1.1370747967113455, 1.007765000781145, 0.9145523912616853, 0.8390743254016101, 0.7615077898186029, 0.6843792173338624, 0.6320158419323604, 0.5900143308948815, 0.550751795487684, 0.5037684703042699, 0.4736161081951874, 0.4535521893538639, 0.4358391510862496, 0.4210578321323425, 0.40810041507267314, 0.3968839045479846, 0.3877734415065673, 0.37747110329431527, 0.3721870546903874, 0.36935648798288095, 0.3682570087317776 ], "peak_layer": 1, "peak_normalized": 2.382164866319327, "population_cv": 0.6517494715357345, "rest_mean": 6.217700879292183e-05, "spike_contrast": 0.41579944566424976, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5853165789158082e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013459566980600357, 0.00012879954010713845, 0.00012227393744979054, 0.00011689116945490241, 0.00010692041541915387, 0.00010222572018392384, 9.748146840138361e-05, 9.089083323488012e-05, 8.065953443292528e-05, 7.514455501222983e-05, 6.934277917025611e-05, 6.424632738344371e-05, 5.694014180335216e-05, 5.16734980919864e-05, 4.740888107335195e-05, 4.302626257413067e-05, 3.866838960675523e-05, 3.5709785151993856e-05, 3.3336640626657754e-05, 3.111825208179653e-05, 2.8463628041208722e-05, 2.67599771177629e-05, 2.5626337446738034e-05, 2.4625525838928297e-05, 2.3790360501152463e-05, 2.3058248189045116e-05, 2.2424499547923915e-05, 2.1909745555603877e-05, 2.1327648937585764e-05, 2.102909274981357e-05, 2.086916174448561e-05, 2.080703961837571e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.002919268561527133, 0.0065401275642216206, 0.010618876665830612, 0.015120964497327805, 0.0030634324066340923, 0.007598300464451313, 0.012669830583035946, 0.017725100740790367, 0.0037359180860221386, 0.009108707308769226, 0.015153576619923115, 0.021781234070658684, 0.005531937349587679, 0.0119093032553792, 0.01904802769422531, 0.02626260370016098, 0.005659298039972782, 0.013823030516505241, 0.02223494090139866, 0.0319429449737072, 0.007882392033934593, 0.01784195750951767, 0.028768405318260193, 0.040818072855472565, 0.012831038795411587, 0.023826664313673973, 0.03824474662542343, 0.053081437945365906, 0.013431218452751637, 0.03125866502523422, 0.04969283565878868, 0.06860727071762085 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.57084181182654e-05, "first_to_last_ratio": 3.4725850785839505, "imbalance_abs_log_ratio": 1.2448992960183491, "last_quartile_mean": 4.523551695001515e-06, "mean": 8.68330698722275e-06, "normalized": [ 2.2299043393712443, 2.1042761707719917, 1.976933189224246, 1.8613268098100455, 1.7187475287766918, 1.6201752163443084, 1.525403311499459, 1.435524363342176, 1.3017427682121105, 1.2100915045221226, 1.1332349539879822, 1.0535572156473645, 0.9690912612097308, 0.9035997979344527, 0.843793758654846, 0.7926704001101033, 0.7497625268960744, 0.7082004131597335, 0.6752277610326668, 0.6475699083216085, 0.6244433954092409, 0.6007951696153412, 0.5813694559888941, 0.5649745160906161, 0.5550358385641657, 0.5437760679691019, 0.5335476779691889, 0.5248102179054668, 0.5123899215671036, 0.50517146191784, 0.49925770343476866, 0.49359537473931375 ], "peak_layer": 1, "peak_normalized": 2.2299043393712443, "population_cv": 0.5276847791721199, "rest_mean": 9.350114733361235e-06, "spike_contrast": 0.543581047186555, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.082545158074936e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.9362943930900656e-05, 1.8272075976710767e-05, 1.716631777526345e-05, 1.61624720931286e-05, 1.4924412425898481e-05, 1.4068478776607662e-05, 1.3245545233075973e-05, 1.2465098734537605e-05, 1.1303432074782904e-05, 1.0507596016395837e-05, 9.840226994128898e-06, 9.148360732069705e-06, 8.414916919718962e-06, 7.846234439057298e-06, 7.32692024030257e-06, 6.883000423840713e-06, 6.510418188554468e-06, 6.149521595943952e-06, 5.863209935341729e-06, 5.6230483096442185e-06, 5.4222336984821595e-06, 5.2168888942105696e-06, 5.0482094593462534e-06, 4.905847163172439e-06, 4.819546575163258e-06, 4.721774530480616e-06, 4.632958280126331e-06, 4.557088232104434e-06, 4.449238986126147e-06, 4.3865588850167114e-06, 4.33520790465991e-06, 4.286040166334715e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.106540149630746e-05, "first_to_last_ratio": 3.3966554168443395, "imbalance_abs_log_ratio": 1.2227912465426896, "last_quartile_mean": 1.7978097275772598e-05, "mean": 3.421876306219929e-05, "normalized": [ 2.125681514851671, 2.035741091532698, 1.9424547124055425, 1.8443747880483625, 1.7129915891324252, 1.6251025199478857, 1.537504861060443, 1.452620900762345, 1.3158482592376062, 1.2267970572702998, 1.1490758791488886, 1.0695328816728442, 0.9835696082276737, 0.9163259113865182, 0.85522266858747, 0.8029399296205194, 0.7578942872402048, 0.7156133216460457, 0.6817566226795772, 0.6538643195822892, 0.6299348137903584, 0.6059197773220268, 0.5863493060398601, 0.5697864420541977, 0.5595343492561913, 0.5483692204744319, 0.5380517969111698, 0.5293376615585488, 0.5166827361721865, 0.509484072574387, 0.5036056858005552, 0.49803141400477713 ], "peak_layer": 1, "peak_normalized": 2.125681514851671, "population_cv": 0.512158847941706, "rest_mean": 3.6814922000134914e-05, "spike_contrast": 0.5486771042805106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.019950479734689e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.273819210240617e-05, 6.966054206714034e-05, 6.646839756285772e-05, 6.311222387012094e-05, 5.861645331606269e-05, 5.560899808187969e-05, 5.261151454760693e-05, 4.9706890422385186e-05, 4.502669980865903e-05, 4.1979477828135714e-05, 3.931995524908416e-05, 3.659809226519428e-05, 3.365653537912294e-05, 3.1355539249489084e-05, 2.926466186181642e-05, 2.7475611204863526e-05, 2.5934205041266978e-05, 2.4487402697559446e-05, 2.332886833755765e-05, 2.2374428226612508e-05, 2.1555590137722902e-05, 2.073382529488299e-05, 2.006414797506295e-05, 1.9497387256706133e-05, 1.9146573322359473e-05, 1.8764516426017508e-05, 1.841146695369389e-05, 1.8113280020770617e-05, 1.7680244127404876e-05, 1.7433914763387293e-05, 1.7232763639185578e-05, 1.704201895336155e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05415340978652239, "first_to_last_ratio": 3.3966554118601855, "imbalance_abs_log_ratio": 1.2227912450753184, "last_quartile_mean": 0.015943156788125634, "mean": 0.030345541599672288, "normalized": [ 2.1256815759027154, 2.0357410833262275, 1.9424548253937048, 1.8443746973080621, 1.712991596235519, 1.6251025854549026, 1.5375048897542698, 1.4526208560186746, 1.3158482418228976, 1.2267970397813441, 1.1490758214573777, 1.0695328097128658, 0.9835695720716091, 0.9163258756382576, 0.8552226910646754, 0.8029399206669349, 0.7578942982897275, 0.715613345638086, 0.6817566530269693, 0.6538643082892013, 0.6299347939093796, 0.6059197755459048, 0.5863493070405259, 0.5697864549709054, 0.5595343861542013, 0.5483692109047464, 0.5380518325125886, 0.5293376691592189, 0.5166827726726805, 0.5094840492707883, 0.5036056657399419, 0.49803139526509604 ], "peak_layer": 1, "peak_normalized": 2.1256815759027154, "population_cv": 0.51215885272173, "rest_mean": 0.03264784129957358, "spike_contrast": 0.5486771102516746, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01791312322020531, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0645049586892128, 0.06177566573023796, 0.05894484370946884, 0.055968549102544785, 0.05198165774345398, 0.04931461811065674, 0.04665641859173775, 0.04408056661486626, 0.03993012756109238, 0.0372278206050396, 0.03486932814121246, 0.032455552369356155, 0.029846951365470886, 0.02780640497803688, 0.02595219574868679, 0.02436564676463604, 0.022998712956905365, 0.021715674549341202, 0.02068827487528324, 0.019841866567730904, 0.019115712493658066, 0.01838696375489235, 0.017793087288737297, 0.017290478572249413, 0.01697937399148941, 0.01664056070148945, 0.016327474266290665, 0.016063038259744644, 0.015679018571972847, 0.01546056941151619, 0.015282186679542065, 0.015113032422959805 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.484899475050042e-06, "first_to_last_ratio": 4.317013540640096, "imbalance_abs_log_ratio": 1.4625638531540355, "last_quartile_mean": 1.2705309870852943e-06, "mean": 2.7753340354763623e-06, "normalized": [ 2.6423318625200567, 2.4162339509651107, 2.211337495963179, 2.033398824132887, 1.8430633472212699, 1.6940067504930827, 1.5445742745339275, 1.4254731551929667, 1.2827132729228703, 1.1755637695304801, 1.092360755278899, 1.0016920978710038, 0.9227083034090567, 0.8576286815024052, 0.7936258208713879, 0.7371254703657203, 0.6937452548738572, 0.6483007348957919, 0.6136555742307378, 0.5846345106553689, 0.5617683465972004, 0.5391257192612848, 0.5197876012499301, 0.5027932393500675, 0.49022455226643835, 0.4787926786546725, 0.46965294739774904, 0.460864066796148, 0.45018919494858056, 0.44356620781452427, 0.43718650171280415, 0.43187503652054116 ], "peak_layer": 1, "peak_normalized": 2.6423318625200567, "population_cv": 0.6359510157609364, "rest_mean": 3.020622224775488e-06, "spike_contrast": 0.48029104777209103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4507778132610838e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.333353551075561e-06, 6.705856321786996e-06, 6.137200216471683e-06, 5.643360964313615e-06, 5.115116437082179e-06, 4.701434590970166e-06, 4.28670955443522e-06, 3.956164164264919e-06, 3.5599578041001223e-06, 3.2625821404508315e-06, 3.031665983144194e-06, 2.7800301722891163e-06, 2.560823759267805e-06, 2.380206069574342e-06, 2.2025767520972295e-06, 2.0457694063225063e-06, 1.9253748178016394e-06, 1.7992510947806295e-06, 1.7030992012223578e-06, 1.622556055735913e-06, 1.559094812364492e-06, 1.496253958066518e-06, 1.4425842209675466e-06, 1.3954191899756552e-06, 1.3605368849312072e-06, 1.3288096170072095e-06, 1.3034438097747625e-06, 1.2790517303074012e-06, 1.249425395144499e-06, 1.2310443935348303e-06, 1.2133385780543904e-06, 1.1985974879280548e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.00092546644737e-05, "first_to_last_ratio": 2.8610718587446584, "imbalance_abs_log_ratio": 1.0511963304057041, "last_quartile_mean": 1.7479202597314725e-05, "mean": 3.0017626613698667e-05, "normalized": [ 1.9665203436980145, 1.8844342899401736, 1.80165076038572, 1.7146572887379672, 1.6027118780052416, 1.5258449550389437, 1.4518445303422347, 1.3803062982231078, 1.2718602647538741, 1.1969127030718432, 1.1317702815894917, 1.0650354822632313, 0.9973554850676779, 0.9413624795803887, 0.8896480124159175, 0.8445590199428843, 0.809061501322106, 0.7713306058151453, 0.7401513167984001, 0.7139298020909397, 0.6922990839843121, 0.6685043123895319, 0.6485424460151992, 0.631323216874115, 0.6206922556356658, 0.6084815208074952, 0.5970366156159529, 0.5872145081510212, 0.5725964819678343, 0.5642525868906152, 0.5573685431962108, 0.5507411293887441 ], "peak_layer": 1, "peak_normalized": 1.9665203436980145, "population_cv": 0.4378481821603461, "rest_mean": 3.195058242353197e-05, "spike_contrast": 0.6128109021943274, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9579665240598844e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.903027340536937e-05, 5.6566244893474504e-05, 5.4081279813544825e-05, 5.14699422637932e-05, 4.810960672330111e-05, 4.580224413075484e-05, 4.358092701295391e-05, 4.1433519072597846e-05, 3.8178226532181725e-05, 3.592847861000337e-05, 3.397305772523396e-05, 3.196983743691817e-05, 2.9938244551885873e-05, 2.825746742018964e-05, 2.670512185432017e-05, 2.5351657313876785e-05, 2.4286106054205447e-05, 2.3153514121077023e-05, 2.2217585865291767e-05, 2.1430478227557614e-05, 2.0781175408046693e-05, 2.006691283895634e-05, 1.9467704987619072e-05, 1.895082459668629e-05, 1.8631708371685818e-05, 1.8265171092934906e-05, 1.7921622202266008e-05, 1.7626785847824067e-05, 1.7187987396027893e-05, 1.693752346909605e-05, 1.673088081588503e-05, 1.6531941582798027e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.890059224431752e-05, "first_to_last_ratio": 2.166537206678912, "imbalance_abs_log_ratio": 0.7731301356847547, "last_quartile_mean": 1.7955192333829473e-05, "mean": 2.5870607089473197e-05, "normalized": [ 1.7831451783784935, 1.6977459463469209, 1.6200768511580206, 1.5334592693492033, 1.446421368117004, 1.3744004997462946, 1.3169700168196983, 1.2570593821131775, 1.1875487586493927, 1.1204242107881341, 1.071956417312422, 1.0193758496059768, 0.9782769348268451, 0.9375232472756838, 0.9065756301529028, 0.8773601959197916, 0.8648597359592212, 0.8401947622081907, 0.814617151100937, 0.7925804041893227, 0.77941283303955, 0.7586878792831788, 0.7416534116249156, 0.7273676136251142, 0.7223613266422326, 0.7135679461862469, 0.7058096849258172, 0.6989156870230683, 0.6866714405082665, 0.6814500720405863, 0.6750012133598903, 0.6685290817235002 ], "peak_layer": 1, "peak_normalized": 1.7831451783784935, "population_cv": 0.3297373433822509, "rest_mean": 2.7087979105999693e-05, "spike_contrast": 0.7123749662061754, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9296798200230114e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.61310482933186e-05, 4.392171831568703e-05, 4.19123716710601e-05, 3.967152224504389e-05, 3.741979890037328e-05, 3.5556575312511995e-05, 3.407081385375932e-05, 3.2520889362785965e-05, 3.0722607334610075e-05, 2.8986054530832916e-05, 2.7732163289329037e-05, 2.6371872081654146e-05, 2.5308618205599487e-05, 2.4254295567516237e-05, 2.3453661924577318e-05, 2.2697840904584154e-05, 2.2374446416506544e-05, 2.1736348571721464e-05, 2.107464024447836e-05, 2.0504536223597825e-05, 2.016388316405937e-05, 1.962771602848079e-05, 1.9187024008715525e-05, 1.8817441741703078e-05, 1.8687926058191806e-05, 1.8460435967426747e-05, 1.825972503866069e-05, 1.808137312764302e-05, 1.776460703695193e-05, 1.762952706485521e-05, 1.7462691175751388e-05, 1.7295253201154992e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011785902643168811, "first_to_last_ratio": 5.2723364637525565, "imbalance_abs_log_ratio": 1.6624736160576234, "last_quartile_mean": 2.235423085039656e-05, "mean": 5.981010565392353e-05, "normalized": [ 2.341513132793844, 2.2440750819699944, 2.142476081227823, 2.0324729445905114, 1.8907305822050442, 1.7814901971753774, 1.70505397052119, 1.6266178018526998, 1.4513520805328384, 1.3486545714944895, 1.25655955310435, 1.1590271011245943, 1.0440203382713282, 0.9503717152908809, 0.8638113904441559, 0.7921778834221871, 0.7124380420896897, 0.6459278658421956, 0.5962877600798272, 0.5569469907381962, 0.5166814306914113, 0.4757859010752184, 0.4474223682075307, 0.42807794732183224, 0.41212661540589873, 0.3977379814905876, 0.3854116965392957, 0.3748299777949146, 0.36298713381472003, 0.3565353744030076, 0.3515636207422689, 0.3488348677420969 ], "peak_layer": 1, "peak_normalized": 2.341513132793844, "population_cv": 0.6457650964234652, "rest_mean": 6.583521119157215e-05, "spike_contrast": 0.4142849283380522, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7274535750620998e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001400461478624493, 0.00013421836774796247, 0.00012814172077924013, 0.00012156242155469954, 0.00011308479588478804, 0.00010655111691448838, 0.00010197945812251419, 9.728818258736283e-05, 8.68055212777108e-05, 8.066317241173238e-05, 7.515495963161811e-05, 6.932153337402269e-05, 6.244296673685312e-05, 5.684183270204812e-05, 5.1664650527527556e-05, 4.7380242904182523e-05, 4.2610994569258764e-05, 3.863301390083507e-05, 3.566403393051587e-05, 3.3311058359686285e-05, 3.090277095907368e-05, 2.845680501195602e-05, 2.6760379114421085e-05, 2.5603387257433496e-05, 2.4649336410220712e-05, 2.3788750695530325e-05, 2.305151429027319e-05, 2.2418620574171655e-05, 2.1710298824473284e-05, 2.1324418412405066e-05, 2.1027057300671004e-05, 2.0863850295427255e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.005866481456905603, 0.010142817161977291, 0.01482025533914566, 0.01956234686076641, 0.0067119961604475975, 0.011908270418643951, 0.017094247043132782, 0.02295343205332756, 0.007494525518268347, 0.013766046613454819, 0.02038806490600109, 0.02682213857769966, 0.009193498641252518, 0.016781754791736603, 0.024297412484884262, 0.03140362724661827, 0.009884056635200977, 0.018740201368927956, 0.02799132652580738, 0.03797563910484314, 0.012623735703527927, 0.02351362258195877, 0.034899674355983734, 0.047332651913166046, 0.01744839735329151, 0.029514454305171967, 0.044660378247499466, 0.0601838044822216, 0.018972838297486305, 0.0376126803457737, 0.0565216988325119, 0.07583927363157272 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5008519199000148e-05, "first_to_last_ratio": 3.366911210627585, "imbalance_abs_log_ratio": 1.2139957693612529, "last_quartile_mean": 4.457652210021479e-06, "mean": 8.299272167278104e-06, "normalized": [ 2.256857417490706, 2.1152794126450303, 1.976744865869958, 1.8670032822761609, 1.7142943873365317, 1.61284428729019, 1.5135327831047092, 1.4107548139296946, 1.2736784355647999, 1.19029358022618, 1.1070446131205842, 1.032219511002358, 0.948055380482258, 0.88484028406668, 0.831196979445672, 0.7835724276747893, 0.7420372577590195, 0.7071991859856492, 0.678238718639715, 0.6514579423864029, 0.6287823606114806, 0.6086317246696183, 0.5913247143847147, 0.5772065844972074, 0.5691335632017953, 0.558269353765152, 0.5491862116730458, 0.5409941785864528, 0.5285468164414351, 0.5224629687441996, 0.516466078557371, 0.5118498785704394 ], "peak_layer": 1, "peak_normalized": 2.256857417490706, "population_cv": 0.5242114468317668, "rest_mean": 8.921693312947389e-06, "spike_contrast": 0.5535045654951547, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.938197980663972e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8730273950495757e-05, 1.7555279555381276e-05, 1.6405543647124432e-05, 1.5494768376811408e-05, 1.4227395695343148e-05, 1.3385433703660965e-05, 1.256122050108388e-05, 1.1708238162100315e-05, 1.0570603990345262e-05, 9.878570381260943e-06, 9.187664545606822e-06, 8.566670658183284e-06, 7.868169632274657e-06, 7.343530342041049e-06, 6.8983299570390955e-06, 6.5030808400479145e-06, 6.158369160402799e-06, 5.869238520972431e-06, 5.628887720376952e-06, 5.406626769399736e-06, 5.218435944698285e-06, 5.051200332673034e-06, 4.907564743916737e-06, 4.7903945414873306e-06, 4.723394340544473e-06, 4.63322930954746e-06, 4.5578458411910105e-06, 4.489857929002028e-06, 4.386553882795852e-06, 4.336062374932226e-06, 4.286292551114457e-06, 4.247981451044325e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.8205007462674985e-05, "first_to_last_ratio": 3.284886533563498, "imbalance_abs_log_ratio": 1.189332110567881, "last_quartile_mean": 1.7719031347951386e-05, "mean": 3.265063452317918e-05, "normalized": [ 2.145023601546889, 2.046116178200582, 1.9398041355219169, 1.8522906626530224, 1.7094551367874427, 1.616905863157966, 1.5255793277696308, 1.4261117891382666, 1.2879069358899493, 1.2060838643982497, 1.1221569343801026, 1.0481379278736214, 0.9613520366262555, 0.8968015077026028, 0.8421366180143057, 0.7933562257498653, 0.7503570432885839, 0.714837243906681, 0.6855582118161445, 0.6585243205638621, 0.6351846609275962, 0.6146934889106834, 0.5972735915655352, 0.5828673640378894, 0.5748017373594797, 0.5639401582431183, 0.5548451094205582, 0.5465856652489405, 0.5340146592968225, 0.5278533354505021, 0.5219685037625007, 0.5174761607904339 ], "peak_layer": 1, "peak_normalized": 2.145023601546889, "population_cv": 0.5072011380661324, "rest_mean": 3.5063370281325966e-05, "spike_contrast": 0.5596113913680666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.962186142918654e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.003638165770099e-05, 6.680699152639136e-05, 6.333583587547764e-05, 6.047846545698121e-05, 5.581479490501806e-05, 5.2793002396356314e-05, 4.981113306712359e-05, 4.656345481635071e-05, 4.2050978663610294e-05, 3.9379403460770845e-05, 3.663913594209589e-05, 3.422236841288395e-05, 3.138875399599783e-05, 2.928113826783374e-05, 2.7496294933371246e-05, 2.5903584173647687e-05, 2.449963358230889e-05, 2.3339889594353735e-05, 2.2383910618373193e-05, 2.1501236915355548e-05, 2.0739182218676433e-05, 2.0070132450200617e-05, 1.950136174855288e-05, 1.9030989278689958e-05, 1.87676414498128e-05, 1.8413003999739885e-05, 1.8116044884664007e-05, 1.7846368791651912e-05, 1.7435917470720597e-05, 1.7234746337635443e-05, 1.7042602848960087e-05, 1.689592500042636e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05161678325384855, "first_to_last_ratio": 3.2848865977727097, "imbalance_abs_log_ratio": 1.189332130114739, "last_quartile_mean": 0.015713414060883224, "mean": 0.028954909328604117, "normalized": [ 2.1450237245911468, 2.046116217659094, 1.9398041664812302, 1.8522906413119131, 1.7094550632279355, 1.6169058509968128, 1.5255792788818974, 1.4261118508948372, 1.2879069434477655, 1.206083847584953, 1.1221569306992827, 1.0481379275281273, 0.9613519652186596, 0.8968015075628347, 0.8421366414011558, 0.7933562213086081, 0.7503569989748414, 0.7148372643644912, 0.6855582307408538, 0.6585243493162974, 0.6351846345398151, 0.614693479533182, 0.5972735918667607, 0.5828673969375834, 0.5748016775305409, 0.5639401482692175, 0.5548451622169414, 0.5465856833639338, 0.5340146386986252, 0.5278533199019263, 0.5219685200223859, 0.5174761249263505 ], "peak_layer": 1, "peak_normalized": 2.1450237245911468, "population_cv": 0.507201147039954, "rest_mean": 0.031094547602589482, "spike_contrast": 0.5596113785438716, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017400862649083138, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06210896745324135, 0.05924510955810547, 0.05616685375571251, 0.05363290756940842, 0.04949711635708809, 0.0468173623085022, 0.04417300969362259, 0.0412929393351078, 0.03729122877120972, 0.034922048449516296, 0.03249195218086243, 0.03034873865544796, 0.02783585898578167, 0.025966806337237358, 0.024383990094065666, 0.022971557453274727, 0.021726518869400024, 0.020698048174381256, 0.01985027641057968, 0.019067512825131416, 0.018391713500022888, 0.017798393964767456, 0.017294002696871758, 0.01687687262892723, 0.016643330454826355, 0.016328835859894753, 0.01606549136340618, 0.015826338902115822, 0.015462345443665981, 0.015283945016562939, 0.015113551169633865, 0.014983474276959896 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.35499910370163e-06, "first_to_last_ratio": 4.280329422122994, "imbalance_abs_log_ratio": 1.4540299744175837, "last_quartile_mean": 1.251071722663255e-06, "mean": 2.683941740855289e-06, "normalized": [ 2.703190107906874, 2.4622030057009887, 2.2387169403666407, 2.052519103467962, 1.857495181173395, 1.6956946287041017, 1.543681691896796, 1.4080949959710705, 1.2505201784095583, 1.154791072279902, 1.0675304639467325, 0.9762128756212061, 0.8974710865803879, 0.8318006025975303, 0.7746066644618449, 0.7211640498251165, 0.6781926692188598, 0.6399504079116765, 0.6084710506699328, 0.581158338700675, 0.5581318067356622, 0.5402181590129327, 0.5217735093776118, 0.5073536475319059, 0.4964690402033343, 0.48529694803264734, 0.47725139647875997, 0.4696126527079396, 0.4585674232309517, 0.45305527095834874, 0.4466379671326055, 0.4421670631860494 ], "peak_layer": 1, "peak_normalized": 2.703190107906874, "population_cv": 0.645765617676911, "rest_mean": 2.9201339657447534e-06, "spike_contrast": 0.48234216065938457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4085037264521817e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.255204764078371e-06, 6.608409421460237e-06, 6.008585842209868e-06, 5.508841695700539e-06, 4.985408850188833e-06, 4.551145593723049e-06, 4.143151727475924e-06, 3.779244934776216e-06, 3.3563233046152163e-06, 3.099391960859066e-06, 2.8651895718212472e-06, 2.6200984848401276e-06, 2.408760110483854e-06, 2.232504357380094e-06, 2.078999159493833e-06, 1.9355622953298735e-06, 1.8202296132585616e-06, 1.7175896118715173e-06, 1.6331008509951062e-06, 1.5597951232848573e-06, 1.497993252996821e-06, 1.44991406614281e-06, 1.400409701091121e-06, 1.3617076319860644e-06, 1.3324939800440916e-06, 1.3025087355345022e-06, 1.2809149438908207e-06, 1.2604130006366177e-06, 1.2307682482060045e-06, 1.2159739526396152e-06, 1.1987502830379526e-06, 1.186750637316436e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.7860596623650054e-05, "first_to_last_ratio": 2.779352348042467, "imbalance_abs_log_ratio": 1.022217932231161, "last_quartile_mean": 1.7220053678101976e-05, "mean": 2.88126707914671e-05, "normalized": [ 1.9752636229242324, 1.8880994931589683, 1.7945585688576082, 1.7155011651574092, 1.5967516750805444, 1.5186230631712434, 1.44161945600051, 1.358346630504372, 1.2494545051794677, 1.1811062374956933, 1.1110457151250825, 1.0496228553992981, 0.981726104768848, 0.9275032753698961, 0.8805136203767746, 0.8381981012329272, 0.8039749772857018, 0.7714588681042523, 0.7440936013254396, 0.7183075709964174, 0.6966458004631945, 0.6758729225629136, 0.6578613567984469, 0.6426061157750497, 0.6340386554461201, 0.6220565974014273, 0.6118747994647586, 0.6024343556608059, 0.5879125773227442, 0.5807482794696509, 0.5737976188800432, 0.5683818132401579 ], "peak_layer": 1, "peak_normalized": 1.9752636229242324, "population_cv": 0.4317074018350235, "rest_mean": 3.0619305444243194e-05, "spike_contrast": 0.62238001123762, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9056843666476197e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.6912620493676513e-05, 5.440118911792524e-05, 5.170602526050061e-05, 4.942817031405866e-05, 4.600668034981936e-05, 4.375558637548238e-05, 4.1536906792316586e-05, 3.9137594285421073e-05, 3.6000121326651424e-05, 3.4030825190711766e-05, 3.201219442416914e-05, 3.0242437787819654e-05, 2.8286151064094156e-05, 2.672384653124027e-05, 2.5369949071318842e-05, 2.4150725948857144e-05, 2.3164666345110163e-05, 2.222779039584566e-05, 2.1439323973027058e-05, 2.0696359570138156e-05, 2.00722261070041e-05, 1.9473704014671966e-05, 1.8954742699861526e-05, 1.85151984624099e-05, 1.8268347048433498e-05, 1.7923111954587512e-05, 1.762974716257304e-05, 1.7357742763124406e-05, 1.6939331544563174e-05, 1.6732908989069983e-05, 1.653264189371839e-05, 1.6376598068745807e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.7207921195658855e-05, "first_to_last_ratio": 2.0918054721335677, "imbalance_abs_log_ratio": 0.7380275553273271, "last_quartile_mean": 1.7787467186280992e-05, "mean": 2.50158799417477e-05, "normalized": [ 1.7669834832946238, 1.6851855745889466, 1.5913246601136481, 1.5163874050648884, 1.4314556947301431, 1.3660671618942286, 1.3038144666622677, 1.2377581340639199, 1.1604947777085126, 1.1096820799312197, 1.0568639468917405, 1.0113831764859333, 0.9697099699695068, 0.9379529791327526, 0.9077702331627978, 0.8815756711399201, 0.8691427723616427, 0.8426692090660671, 0.8202682499619266, 0.8014514792168752, 0.7847031862420706, 0.7670164542718427, 0.7519889963555914, 0.7399739814225563, 0.7380541298189486, 0.7300354427325324, 0.7228094770104511, 0.7167792102532495, 0.7048467977298535, 0.6982014484170743, 0.6912475034823692, 0.6864022468218989 ], "peak_layer": 1, "peak_normalized": 1.7669834832946238, "population_cv": 0.3174743127828502, "rest_mean": 2.6144617987921703e-05, "spike_contrast": 0.7236936680868339, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8920694492408076e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.420264667714946e-05, 4.21564000134822e-05, 3.9808386645745486e-05, 3.793376527028158e-05, 3.58091238013003e-05, 3.417337211431004e-05, 3.26160661643371e-05, 3.096360887866467e-05, 2.9030798032181337e-05, 2.7759673685068265e-05, 2.6438381610205397e-05, 2.5300640118075535e-05, 2.4258148187072948e-05, 2.3463719116989523e-05, 2.2708671167492867e-05, 2.2053391148801893e-05, 2.1742371245636605e-05, 2.1080111764604226e-05, 2.0519732061075047e-05, 2.004901398322545e-05, 1.9630040696938522e-05, 1.918759153340943e-05, 1.881166645034682e-05, 1.851110027928371e-05, 1.846307350206189e-05, 1.826247898861766e-05, 1.8081715097650886e-05, 1.793086266843602e-05, 1.763236286933534e-05, 1.746612360875588e-05, 1.729216455714777e-05, 1.7170956198242493e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011250466286583105, "first_to_last_ratio": 5.136757236447842, "imbalance_abs_log_ratio": 1.6364219925229777, "last_quartile_mean": 2.1901884338149102e-05, "mean": 5.649989418543555e-05, "normalized": [ 2.3823389400332498, 2.2792913554127336, 2.1636479699592535, 2.0682892196556057, 1.89270947379848, 1.8094109355775225, 1.7253047001597102, 1.6089010079061294, 1.4276863120740728, 1.3301882974473787, 1.227270134970172, 1.1372581289133685, 1.0077700511385481, 0.9144159274954295, 0.8389896038222593, 0.761607782391717, 0.6842191357138161, 0.6319663297187056, 0.5899873238938539, 0.550774281755648, 0.5037238917813971, 0.4736966989455727, 0.453500437871209, 0.43589448694769534, 0.4210071093140732, 0.4081032387076438, 0.39686588746098733, 0.38782343274226294, 0.37750089000360326, 0.37216747430584624, 0.36939626290542954, 0.36829327717662536 ], "peak_layer": 1, "peak_normalized": 2.3823389400332498, "population_cv": 0.6517280428580817, "rest_mean": 6.217536584127056e-05, "spike_contrast": 0.4157972678427941, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5852347243926487e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001346018980257213, 0.00012877972039859742, 0.00012224588135723025, 0.00011685812205541879, 0.00010693788499338552, 0.00010223152639809996, 9.747953299665824e-05, 9.090273670153692e-05, 8.066412556217983e-05, 7.515549805248156e-05, 6.934063276275992e-05, 6.425496394513175e-05, 5.6938901252578944e-05, 5.1664403144968674e-05, 4.740282383863814e-05, 4.3030759115936235e-05, 3.8658308767480776e-05, 3.570603075786494e-05, 3.333422137075104e-05, 3.111868863925338e-05, 2.8460346584324725e-05, 2.676381336641498e-05, 2.5622726752771996e-05, 2.4627992388559505e-05, 2.3786857127561234e-05, 2.3057789803715423e-05, 2.242288064735476e-05, 2.1911982912570238e-05, 2.132876034011133e-05, 2.1027422917541116e-05, 2.08708497666521e-05, 2.080853118968662e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.028571506962180138, 0.01502692885696888, 0.015751885250210762, 0.016715344041585922, 0.01783851347863674, 0.012985615991055965, 0.014028912410140038, 0.015184907242655754, 0.016659293323755264, 0.013609340414404869, 0.014725575223565102, 0.016002535820007324, 0.017315881326794624, 0.014967667870223522, 0.016154561191797256, 0.01737469807267189, 0.018479224294424057, 0.01647813245654106, 0.017671043053269386, 0.01892261393368244, 0.020303834229707718, 0.01861351728439331, 0.01984807848930359, 0.021230308338999748, 0.022779423743486404, 0.02138628251850605, 0.02279111184179783, 0.024287279695272446, 0.02611272968351841, 0.024725768715143204, 0.026407435536384583, 0.028016721829771996 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.961159703498197e-06, "first_to_last_ratio": 10.14195644845838, "imbalance_abs_log_ratio": 2.3166809231867567, "last_quartile_mean": 4.891718603516892e-07, "mean": 2.1833543613070105e-06, "normalized": [ 2.1149003512951285, 3.2694277026187555, 2.6224151681895895, 2.367649688300867, 1.872384189095874, 2.4215723461348544, 1.997727875442672, 1.5120426998182623, 1.407451434597839, 1.4959940566742542, 1.146826747734668, 1.0540980256857684, 0.9488948361068527, 0.9447470595114825, 0.8146834014726949, 0.6963182667169648, 0.5482907339349459, 0.5718154556650352, 0.5280062120840487, 0.46712124843842245, 0.37653152797943845, 0.3855401511265584, 0.3587208946748597, 0.2844717467264502, 0.29151483884188434, 0.260094980728135, 0.263913337696718, 0.20680766564782654, 0.1887856679838017, 0.2117583846844945, 0.19386962063621954, 0.1756236837546329 ], "peak_layer": 2, "peak_normalized": 3.2694277026187555, "population_cv": 0.8462629641613505, "rest_mean": 2.4504692364475717e-06, "spike_contrast": 0.3023641450084502, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.409340355479799e-07, "top_five_layers": [ 2, 3, 6, 4, 1 ], "top_five_spike_overlap": 0 }, "values": [ 4.617576905729948e-06, 7.13831923349062e-06, 5.725661594624398e-06, 5.1694182729988825e-06, 4.0880781853047665e-06, 5.2871505431539845e-06, 4.361747869552346e-06, 3.30132502313063e-06, 3.0729652280570008e-06, 3.2662851481291e-06, 2.5039291813300224e-06, 2.3014695216261316e-06, 2.071773678835598e-06, 2.0627176127163693e-06, 1.7787425576898386e-06, 1.5203095244942233e-06, 1.197112965201086e-06, 1.2484757689890102e-06, 1.1528246659509023e-06, 1.0198912150372053e-06, 8.221017537834996e-07, 8.417707704211352e-07, 7.832148298803077e-07, 6.211026288838184e-07, 6.364801947711385e-07, 5.678795105268364e-07, 5.762163368672191e-07, 4.515344187439041e-07, 4.121860115446907e-07, 4.6234359274421877e-07, 4.2328608174102555e-07, 3.834487358744809e-07 ] }, "element_rms": { "statistics": { "first_quartile_mean": 8.940424550019088e-06, "first_to_last_ratio": 7.362808193451028, "imbalance_abs_log_ratio": 1.9964414079930852, "last_quartile_mean": 1.214268294802423e-06, "mean": 4.16189458363192e-06, "normalized": [ 1.8755411782360838, 2.990879153140948, 2.507234682429089, 2.1623895678054548, 1.7817542090336904, 2.3421906478647685, 1.984613814607898, 1.5406942028635409, 1.3930431107090302, 1.467345705614427, 1.208878006326314, 1.0663129583432038, 0.9225426136071778, 0.9347168647367088, 0.8446162613986855, 0.7123769822844362, 0.5995323166764556, 0.6272780709200748, 0.55800311830824, 0.5090119191108281, 0.4207840547535497, 0.45050976377329044, 0.4129238940687079, 0.3527585690707129, 0.3423379312955225, 0.33392765114492584, 0.32776335752361546, 0.2883552290395252, 0.26514548144992983, 0.28609378158280374, 0.25631521585884554, 0.23412968642151585 ], "peak_layer": 2, "peak_normalized": 2.990879153140948, "population_cv": 0.7649777321045976, "rest_mean": 4.627515910147782e-06, "spike_contrast": 0.3560310655730732, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6475394204462646e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 7.805804671079386e-06, 1.2447723747754935e-05, 1.0434846444695722e-05, 8.99963742995169e-06, 7.415473191940691e-06, 9.747950571181718e-06, 8.259753485617694e-06, 6.412206857930869e-06, 5.797698577225674e-06, 6.106938144512242e-06, 5.0312228268012404e-06, 4.437882125785109e-06, 3.839525106741348e-06, 3.890193056577118e-06, 3.515203843562631e-06, 2.9648379040736472e-06, 2.4951903014880372e-06, 2.610665205793339e-06, 2.3223501557367854e-06, 2.1184539491514442e-06, 1.7512588783574756e-06, 1.8749741457213531e-06, 1.718545718176756e-06, 1.4681439779451466e-06, 1.4247743820305914e-06, 1.3897716826249962e-06, 1.3641165423905477e-06, 1.2001040659015416e-06, 1.1035075431209407e-06, 1.1906921599802445e-06, 1.0667569085853756e-06, 9.744230737851467e-07 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.00792845810065046, "first_to_last_ratio": 7.362808269910068, "imbalance_abs_log_ratio": 1.9964414183775794, "last_quartile_mean": 0.001076825283234939, "mean": 0.003690809797262773, "normalized": [ 1.8755411129192268, 2.990879196958467, 2.5072346853780427, 2.162389622214153, 1.7817542918869094, 2.3421908242130387, 1.9846136654580344, 1.5406942156700558, 1.3930430879772915, 1.4673456922088532, 1.2088779947214396, 1.0663129426719855, 0.9225426097974535, 0.9347168545387744, 0.8446162145203677, 0.7123769861164736, 0.5995323323039541, 0.6272780785788562, 0.5580030712234716, 0.5090119005316229, 0.4207840635329223, 0.45050976081671684, 0.4129238737745274, 0.35275859035228563, 0.3423379200763214, 0.3339276703286944, 0.3277633648171629, 0.2883552364940563, 0.2651454743312869, 0.2860937736207281, 0.25631521203611624, 0.2341296799307093 ], "peak_layer": 2, "peak_normalized": 2.990879196958467, "population_cv": 0.7649777429861212, "rest_mean": 0.004103727453319287, "spike_contrast": 0.3560310647277095, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0014610544545575976, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.006922265514731407, 0.011038766242563725, 0.009253726340830326, 0.007980968803167343, 0.006576116196811199, 0.008644580841064453, 0.007324831560254097, 0.005686409305781126, 0.005141457077115774, 0.005415693856775761, 0.004461738746613264, 0.003935558255761862, 0.0034049293026328087, 0.0034498621243983507, 0.0031173177994787693, 0.002629247959703207, 0.002212759805843234, 0.00231516407802701, 0.002059483202174306, 0.001878666109405458, 0.0015530339442193508, 0.001662745838984847, 0.0015240234788507223, 0.0013019648613408208, 0.0012635041493922472, 0.0012324635172262788, 0.0012097122380509973, 0.0010642643319442868, 0.0009786015143617988, 0.001055917702615261, 0.0009460106957703829, 0.000864128116518259 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.5133993492308946e-06, "first_to_last_ratio": 6.5659297300766655, "imbalance_abs_log_ratio": 1.881894116932073, "last_quartile_mean": 5.350954843663658e-07, "mean": 1.7443735549349526e-06, "normalized": [ 1.589774202228725, 2.5970667163618275, 2.31809881449688, 2.0431360816541173, 1.739210008196666, 2.257659305838541, 2.011325443479608, 1.5567888778424819, 1.467448725425628, 1.5902279397678998, 1.2912623477212033, 1.1632146422568665, 1.0012751149296444, 1.0140542592766522, 0.8991815616774139, 0.774122319459461, 0.6409300827695702, 0.6678856249973933, 0.6105498714134762, 0.5542747505087504, 0.4535313629747497, 0.4812253529456957, 0.4453558428285356, 0.3783599505586694, 0.3693486135392654, 0.34798394288991763, 0.34559725866794794, 0.2988109680432934, 0.27684960189841046, 0.29841376846953566, 0.272740676748336, 0.2442959701328381 ], "peak_layer": 2, "peak_normalized": 2.5970667163618275, "population_cv": 0.6989714208275406, "rest_mean": 1.9299347726515677e-06, "spike_contrast": 0.38464666774479367, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.423429792652314e-07, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 2.773160076685599e-06, 4.530254500423325e-06, 4.043630269734422e-06, 3.5639925499708625e-06, 3.033831944776466e-06, 3.938201189157553e-06, 3.5085029139736434e-06, 2.7156213491252856e-06, 2.559778749855468e-06, 2.773951564449817e-06, 2.2524438918480882e-06, 2.0290808606659994e-06, 1.746597831697727e-06, 1.7688894331513438e-06, 1.5685085372751928e-06, 1.3503585023499909e-06, 1.1180214869455085e-06, 1.1650420219666557e-06, 1.0650270496626035e-06, 9.66862216955633e-07, 7.911281159067585e-07, 8.394367796427105e-07, 7.768669547658646e-07, 6.600010920010391e-07, 6.442819540097844e-07, 6.070139875191671e-07, 6.028507186783827e-07, 5.212379505792342e-07, 4.829291242458567e-07, 5.205450861467398e-07, 4.7576162387485965e-07, 4.261434298769018e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.804085307341666e-06, "first_to_last_ratio": 6.304609762674019, "imbalance_abs_log_ratio": 1.8412810741903929, "last_quartile_mean": 7.619956647886283e-07, "mean": 2.3993721356418973e-06, "normalized": [ 1.4988203443996868, 2.5234715181763763, 2.2553217408275312, 1.9958915618525483, 1.7576503840860649, 2.319660501024385, 2.0260662582488056, 1.6409258036937564, 1.4769401358690522, 1.5639642180334559, 1.318106777079787, 1.1580760773918135, 0.96371832012835, 1.0214962965942758, 0.8943161393616675, 0.7727506438550246, 0.6438659223631747, 0.6719055424540622, 0.6043445279969172, 0.5577548422477306, 0.4603517050297865, 0.4900258115330936, 0.45428577798054076, 0.38963894059637383, 0.37546196475112736, 0.3630494912829247, 0.35422949063900455, 0.315438725345089, 0.2890783675728055, 0.3094773464831404, 0.27956031292056627, 0.2543545101810816 ], "peak_layer": 2, "peak_normalized": 2.5234715181763763, "population_cv": 0.6891965904370858, "rest_mean": 2.650882836237374e-06, "spike_contrast": 0.3927802233252255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.041214352426323e-06, "top_five_layers": [ 2, 6, 3, 7, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.5962277706858004e-06, 6.054747245798353e-06, 5.411356141848955e-06, 4.788886599271791e-06, 4.217257355776383e-06, 5.565728770307032e-06, 4.861286925006425e-06, 3.937191650038585e-06, 3.5437290080153616e-06, 3.752532165890443e-06, 3.1626286727259867e-06, 2.7786554710473865e-06, 2.312318883923581e-06, 2.4509497507096967e-06, 2.145797225239221e-06, 1.8541163626650814e-06, 1.5448739532075706e-06, 1.6121514363476308e-06, 1.4500474208034575e-06, 1.3382614270085469e-06, 1.1045550536437077e-06, 1.1757542779378127e-06, 1.090000637304911e-06, 9.348888170279679e-07, 9.008729762172152e-07, 8.710908332432155e-07, 8.499283694618498e-07, 7.56854888095404e-07, 6.936065801710356e-07, 7.4255132176404e-07, 6.707692250529362e-07, 6.102911243033304e-07 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.828203619742453e-06, "first_to_last_ratio": 5.846579859482356, "imbalance_abs_log_ratio": 1.765856850870783, "last_quartile_mean": 4.837364215859452e-07, "mean": 1.4834222348270032e-06, "normalized": [ 1.2646232648739053, 2.247789759160063, 2.020880547315863, 1.842393507943919, 1.7522315558445254, 2.3420641180772317, 2.0534732489830905, 1.7288628769088255, 1.548314882558432, 1.6499586193135367, 1.4182480230650574, 1.2414487105462828, 0.9740236314282787, 1.1015223442115594, 0.9385100939682994, 0.8083610783044318, 0.6755087261126067, 0.7006206946100645, 0.6299107486392663, 0.5832268076063307, 0.47664186001684955, 0.5068198063687983, 0.4772500226016884, 0.4085558916985, 0.3939522492022734, 0.36559718120224766, 0.3572742323275962, 0.3258561263199403, 0.2984295494995466, 0.3178990099824609, 0.2887716969712665, 0.26097913433726316 ], "peak_layer": 6, "peak_normalized": 2.3420641180772317, "population_cv": 0.6540082332746273, "rest_mean": 1.6337852183891368e-06, "spike_contrast": 0.41098555430286204, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.714621235914819e-07, "top_five_layers": [ 6, 2, 7, 3, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.87597026979347e-06, 3.3344213079544716e-06, 2.9978191378177144e-06, 2.7330474949849304e-06, 2.5992992505052825e-06, 3.4742699881462613e-06, 3.0461678761639632e-06, 2.564633632573532e-06, 2.2968047233007383e-06, 2.447585302434163e-06, 2.1038606519141467e-06, 1.8415926206216682e-06, 1.4448883121076506e-06, 1.6340227375621907e-06, 1.3922067410021555e-06, 1.1991407973255264e-06, 1.002064664135105e-06, 1.0393163165645092e-06, 9.34423610488011e-07, 8.651716143504018e-07, 7.070611331982946e-07, 7.518277698181919e-07, 7.079632950990344e-07, 6.06060893915128e-07, 5.843975259267609e-07, 5.42334987585491e-07, 5.299885401655047e-07, 4.833822231375962e-07, 4.4269702925703314e-07, 4.7157845983747393e-07, 4.2837035607590224e-07, 3.8714225070179964e-07 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 1.4976229522289941e-05, "first_to_last_ratio": 6.400592033457801, "imbalance_abs_log_ratio": 1.8563904913150628, "last_quartile_mean": 2.339819417329636e-06, "mean": 7.376071508247151e-06, "normalized": [ 1.5647793422352725, 2.5744682287007357, 2.3807087714141812, 2.046233253632905, 1.770279995956188, 2.2790875337951744, 2.0129055774210696, 1.6145788731579245, 1.4481155829279344, 1.5649856288248702, 1.2953241737825878, 1.1378217116015181, 0.9671807788478195, 0.9743671476553158, 0.8907704628221471, 0.7647048808528004, 0.6263127195536261, 0.6555825424956646, 0.6004321371903539, 0.5418622203213719, 0.44846797056121285, 0.4882635266638759, 0.4317412168950189, 0.383285230653721, 0.3707675618784486, 0.36766574152703, 0.3569460829427414, 0.31124413981080545, 0.2846577996070008, 0.3121781014204342, 0.2794134285368139, 0.25486763631343645 ], "peak_layer": 2, "peak_normalized": 2.5744682287007357, "population_cv": 0.7019462950147708, "rest_mean": 8.162162531435671e-06, "spike_contrast": 0.3836213712934221, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.131179983029142e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 1.154192432295531e-05, 1.8989461750607006e-05, 1.756027813826222e-05, 1.5093162801349536e-05, 1.3057711839792319e-05, 1.681071262282785e-05, 1.484733547840733e-05, 1.1909249224117957e-05, 1.068140409188345e-05, 1.1543445907591376e-05, 9.554403732181527e-06, 8.392654308408964e-06, 7.1339945861836895e-06, 7.187001756392419e-06, 6.5703866312105674e-06, 5.640517883875873e-06, 4.61972740595229e-06, 4.835623713006498e-06, 4.428830379765714e-06, 3.996814484708011e-06, 3.307931820017984e-06, 3.601466687541688e-06, 3.184554088875302e-06, 2.8271392693568487e-06, 2.7348080493538873e-06, 2.7119288006360875e-06, 2.632859832374379e-06, 2.295759031767375e-06, 2.099656285281526e-06, 2.302647999385954e-06, 2.0609734292520443e-06, 1.8799219105858356e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014360599219799042, 0.014914286322891712, 0.015590262599289417, 0.016720104962587357, 0.01203431747853756, 0.01319553330540657, 0.014193638227880001, 0.015358141623437405, 0.012656479142606258, 0.013699201866984367, 0.01495310477912426, 0.01618308201432228, 0.01409733947366476, 0.01521329116076231, 0.016374915838241577, 0.017517324537038803, 0.015613679774105549, 0.016780734062194824, 0.017834074795246124, 0.01904122717678547, 0.017550688236951828, 0.018688000738620758, 0.019847052171826363, 0.021104788407683372, 0.019968591630458832, 0.021149026229977608, 0.02249406836926937, 0.02368815243244171, 0.02278727851808071, 0.024227207526564598, 0.02562388963997364, 0.0269329734146595 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.831441280359286e-06, "first_to_last_ratio": 3.949798443260095, "imbalance_abs_log_ratio": 1.373664550588347, "last_quartile_mean": 1.982744535666825e-06, "mean": 4.823673624088087e-06, "normalized": [ 1.646132756330255, 1.5986846162740493, 1.5570334831325985, 1.4112719107036644, 1.9597644907314193, 1.739595316367215, 1.586190473612319, 1.4896705111709025, 1.6742871420623147, 1.4631400905900616, 1.3644640801393357, 1.1849574810055368, 1.2519074382611204, 1.1640272230786968, 1.006304156490551, 0.9127806011294972, 0.9624471043561035, 0.8599074596436582, 0.7423282252089072, 0.6792100847159988, 0.7315895600277779, 0.6384373973703829, 0.5798636442076341, 0.5076487157583509, 0.5180109917100456, 0.4991175722294294, 0.4463413444888274, 0.40238605877214556, 0.38468977858519854, 0.36602261687556353, 0.35225030635593113, 0.31953736861450793 ], "peak_layer": 5, "peak_normalized": 1.9597644907314193, "population_cv": 0.4957051427675951, "rest_mean": 5.185350830658906e-06, "spike_contrast": 0.5536012513623684, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.870616708605667e-06, "top_five_layers": [ 5, 6, 9, 1, 2 ], "top_five_spike_overlap": 0 }, "values": [ 7.940407158457674e-06, 7.711532816756517e-06, 7.510621344408719e-06, 6.807515092077665e-06, 9.45326428336557e-06, 8.391240044147708e-06, 7.651265150343534e-06, 7.185684353316901e-06, 8.076214726315811e-06, 7.057710263325134e-06, 6.5817293943837285e-06, 5.715848146792268e-06, 6.038792889739852e-06, 5.61488741368521e-06, 4.854082817473682e-06, 4.402955710247625e-06, 4.6425307118624914e-06, 4.147912932239706e-06, 3.580749080356327e-06, 3.2762877708591986e-06, 3.5289492643642006e-06, 3.079613634326961e-06, 2.7970729661319638e-06, 2.448731720505748e-06, 2.49871595769946e-06, 2.407580268481979e-06, 2.1530049707507715e-06, 1.9409790183999576e-06, 1.8556179384177085e-06, 1.7655736428423552e-06, 1.6991405118460534e-06, 1.5413439768963144e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.103702101725503e-05, "first_to_last_ratio": 3.8954992147916703, "imbalance_abs_log_ratio": 1.3598218392207078, "last_quartile_mean": 7.967405281306128e-06, "mean": 1.931944555622067e-05, "normalized": [ 1.6032842527864015, 1.5688395889631517, 1.533796016047905, 1.3882386982091592, 1.9400717090010198, 1.7314283899901333, 1.585633146781874, 1.5008455649726484, 1.675406616182152, 1.4714366899853428, 1.3716626344955323, 1.1913524891080052, 1.2645012175425632, 1.1767240803978434, 1.0170067664778755, 0.9256058574210715, 0.9711818883737663, 0.8683475684259824, 0.7494840741626005, 0.6894104604993242, 0.7371611971197432, 0.6427392384511877, 0.5847748939175224, 0.5118396580731392, 0.5189878230973387, 0.5013130375398106, 0.4483794446409543, 0.404418519263501, 0.386302743223401, 0.36748312424709273, 0.35304323396633686, 0.31929937663562014 ], "peak_layer": 5, "peak_normalized": 1.9400717090010198, "population_cv": 0.4898474013710311, "rest_mean": 2.0753733493890008e-05, "spike_contrast": 0.5576967968781986, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1574290692806243e-05, "top_five_layers": [ 5, 6, 9, 1, 7 ], "top_five_spike_overlap": 0 }, "values": [ 3.0974562832852826e-05, 3.0309111025417224e-05, 2.9632088626385666e-05, 2.682000194909051e-05, 3.74811097572092e-05, 3.345023651490919e-05, 3.063355325139128e-05, 2.8995504180784337e-05, 3.236792690586299e-05, 2.8427341021597385e-05, 2.649976158863865e-05, 2.3016269551590085e-05, 2.44294624280883e-05, 2.2733656805939972e-05, 1.9648006855277345e-05, 1.7882191968965344e-05, 1.876269561762456e-05, 1.677599357208237e-05, 1.4479616766038816e-05, 1.3319027857505716e-05, 1.4241545613913331e-05, 1.2417365724104457e-05, 1.1297526725684293e-05, 9.888458407658618e-06, 1.0026556992670521e-05, 9.68508993537398e-06, 8.662442269269377e-06, 7.813141564838588e-06, 7.46315481592319e-06, 7.099570211721584e-06, 6.820599537604721e-06, 6.168686923047062e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027523940661922097, "first_to_last_ratio": 3.8954991154967806, "imbalance_abs_log_ratio": 1.3598218137310631, "last_quartile_mean": 0.007065574871376157, "mean": 0.017132677690824494, "normalized": [ 1.6032842798207116, 1.5688395364239456, 1.5337959695585364, 1.3882387052917204, 1.940071709225884, 1.7314283707937437, 1.5856331209472507, 1.5008455668619336, 1.6754066305691406, 1.4714366809574604, 1.3716626672028296, 1.1913525502963036, 1.2645012070621602, 1.176724101825654, 1.0170067499274977, 0.9256058277235238, 0.9711818600927834, 0.8683475735699788, 0.7494840504534797, 0.6894104671183356, 0.7371612027997355, 0.6427392553352459, 0.5847748722004306, 0.5118396849118217, 0.5189878410918901, 0.5013130358668625, 0.44837945479745345, 0.40441852259963446, 0.38630275368745787, 0.36748313889002426, 0.3530432360984768, 0.31929937599809327 ], "peak_layer": 5, "peak_normalized": 1.940071709225884, "population_cv": 0.4898473968405949, "rest_mean": 0.018404618560991907, "spike_contrast": 0.5576968062611827, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01026419699192047, "top_five_layers": [ 5, 6, 9, 1, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.027468552812933922, 0.026878422126173973, 0.02627803198993206, 0.023784246295690536, 0.033238623291254044, 0.02966400422155857, 0.02716614119708538, 0.02571350336074829, 0.02870420180261135, 0.02520965039730072, 0.02350025437772274, 0.020411059260368347, 0.021664291620254517, 0.020160434767603874, 0.017424048855900764, 0.015858106315135956, 0.016638945788145065, 0.014877119101583958, 0.01284066867083311, 0.011811447329819202, 0.012629545293748379, 0.011011844500899315, 0.010018759407103062, 0.008769184350967407, 0.008891651406884193, 0.008588834665715694, 0.00768194068223238, 0.006928772199898958, 0.0066184005700051785, 0.0062959701754152775, 0.006048575975000858, 0.005470453295856714 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.0978330894649844e-06, "first_to_last_ratio": 3.7123595485838843, "imbalance_abs_log_ratio": 1.3116676713818918, "last_quartile_mean": 5.650942647150714e-07, "mean": 1.3341314888393185e-06, "normalized": [ 1.5241148267186329, 1.5504517836489524, 1.525800361966804, 1.340018587688921, 1.9164903896693077, 1.732090277095866, 1.5197216980945576, 1.4707803330175575, 1.6808225666055663, 1.5059009907724967, 1.3766280212942543, 1.196783029382642, 1.2615435432700803, 1.196050358398629, 1.046816748775451, 0.9373627935785216, 0.9843932320660254, 0.8775573102072941, 0.7634158116096246, 0.6842432896373823, 0.7549093975240115, 0.6636832533885216, 0.5908781007193045, 0.5110061412689536, 0.5365346292575633, 0.5196779133499698, 0.4612664477302504, 0.4092178350653177, 0.3933915708760512, 0.3701864458066216, 0.3616807560410091, 0.33658155547385904 ], "peak_layer": 5, "peak_normalized": 1.9164903896693077, "population_cv": 0.4775581328679472, "rest_mean": 1.4301389744223006e-06, "spike_contrast": 0.570357903168614, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.156910666912154e-07, "top_five_layers": [ 5, 6, 9, 2, 3 ], "top_five_spike_overlap": 0 }, "values": [ 2.0333695829322096e-06, 2.068506546493154e-06, 2.035618308582343e-06, 1.7877609934657812e-06, 2.5568501769157592e-06, 2.3108361801860156e-06, 2.0275085717003094e-06, 1.9622143554443028e-06, 2.242438313260209e-06, 2.009069930863916e-06, 1.8366027916272287e-06, 1.596665924807894e-06, 1.6830649656185415e-06, 1.5956884453771636e-06, 1.3965911875857273e-06, 1.2505652193794958e-06, 1.3133100082995952e-06, 1.170776840808685e-06, 1.018497073346225e-06, 9.128705187322339e-07, 1.0071483984575025e-06, 8.85440726960951e-07, 7.883090802351944e-07, 6.817493840571842e-07, 7.158077437452448e-07, 6.933186682545056e-07, 6.153900926619826e-07, 5.45950399555295e-07, 5.248360821497045e-07, 4.938773940921237e-07, 4.825296855415218e-07, 4.4904405172019324e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.398965625616256e-05, "first_to_last_ratio": 3.369027618649736, "imbalance_abs_log_ratio": 1.2146241622684435, "last_quartile_mean": 7.120646955627308e-06, "mean": 1.5399188356468585e-05, "normalized": [ 1.5913426512839013, 1.5418831400755482, 1.5021141560062705, 1.3562195006410476, 1.8805635081013792, 1.6560198739348433, 1.512688238851242, 1.4219848670303201, 1.6092469390600324, 1.4165297374033572, 1.3242582904493114, 1.1412781561654184, 1.2305966607275267, 1.1486603349616307, 1.006300256787253, 0.9179301571399356, 0.9843528747997042, 0.8966403626677242, 0.7866772482945813, 0.7196405302926673, 0.7789613145990362, 0.6869940328779693, 0.6281068759775288, 0.5617779475196961, 0.5746692969502841, 0.5501909649183, 0.500671447520295, 0.45822743358408635, 0.4365896209302579, 0.4181501204665046, 0.398078906415543, 0.36265455356680404 ], "peak_layer": 5, "peak_normalized": 1.8805635081013792, "population_cv": 0.4466412974933768, "rest_mean": 1.640840013418778e-05, "spike_contrast": 0.6063628797090791, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.949444756784941e-06, "top_five_layers": [ 5, 6, 9, 1, 2 ], "top_five_spike_overlap": 0 }, "values": [ 2.45053852268029e-05, 2.37437488976866e-05, 2.3131338821258396e-05, 2.088467954308726e-05, 2.8959151677554473e-05, 2.5501361960778013e-05, 2.3294171114685014e-05, 2.1897412807447836e-05, 2.478109672665596e-05, 2.181340823881328e-05, 2.0392502847244032e-05, 1.7574757293914445e-05, 1.895018976938445e-05, 1.7688436855678447e-05, 1.5496207197429612e-05, 1.4135379387880675e-05, 1.5158235328271985e-05, 1.3807533832732588e-05, 1.2114191122236662e-05, 1.108188007492572e-05, 1.1995372005912941e-05, 1.0579150512057822e-05, 9.672336091171019e-06, 8.650924428366125e-06, 8.849440746416803e-06, 8.472494300804101e-06, 7.7099339250708e-06, 7.056330559862545e-06, 6.72312580718426e-06, 6.439172466343734e-06, 6.130092060629977e-06, 5.584585778706241e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7287268860854965e-05, "first_to_last_ratio": 2.7170527496823964, "imbalance_abs_log_ratio": 0.9995477449345839, "last_quartile_mean": 6.362507633639325e-06, "mean": 1.1818911630712137e-05, "normalized": [ 1.5531683121051287, 1.4809645584609277, 1.4249957798471882, 1.2911504637898432, 1.7654831615088196, 1.5114625239638928, 1.4072245252680236, 1.266979301315116, 1.5685685065032353, 1.3590910493816417, 1.257358469917724, 1.0915032257961577, 1.1980939699800082, 1.1009846962306815, 1.0217422520554849, 0.917572741077425, 1.0166318370670364, 0.924735094262937, 0.841915821310778, 0.7538749370193452, 0.8515117201007877, 0.7457322090867645, 0.703238735238229, 0.6393538153043691, 0.6469780758226346, 0.6196156550317132, 0.5947215623716378, 0.5572499311933502, 0.511086746910104, 0.5033639872021801, 0.45209119593165065, 0.4215551389451849 ], "peak_layer": 5, "peak_normalized": 1.7654831615088196, "population_cv": 0.3798574342343211, "rest_mean": 1.2437515848702148e-05, "spike_contrast": 0.6816834612878747, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.478448853566078e-06, "top_five_layers": [ 5, 9, 1, 6, 2 ], "top_five_spike_overlap": 0 }, "values": [ 1.8356759028392844e-05, 1.7503389244666323e-05, 1.6841899196151644e-05, 1.5259993233485147e-05, 2.086608947138302e-05, 1.7863842003862374e-05, 1.6631862308713607e-05, 1.4974316400184762e-05, 1.8538772565079853e-05, 1.6062977010733448e-05, 1.4860608644085005e-05, 1.2900380170322023e-05, 1.4160166756482795e-05, 1.301244083151687e-05, 1.2075881386408582e-05, 1.0844711141544394e-05, 1.2015481843263842e-05, 1.092936236091191e-05, 9.950528692570515e-06, 8.909981261240318e-06, 1.0063941772386897e-05, 8.813743079372216e-06, 8.311516467074398e-06, 7.556466243840987e-06, 7.646576705155894e-06, 7.323182671825634e-06, 7.028961590549443e-06, 6.586087692994624e-06, 6.040489097358659e-06, 5.949214482825482e-06, 5.343225893739145e-06, 4.982322934665717e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.156662675493862e-05, "first_to_last_ratio": 4.432028161582752, "imbalance_abs_log_ratio": 1.488857303572528, "last_quartile_mean": 1.3891298635826388e-05, "mean": 3.828891439638937e-05, "normalized": [ 1.5694774864844574, 1.533915591681154, 1.4779364465001996, 1.3758197901662568, 1.9142257050168354, 1.758664969175181, 1.6344745902568742, 1.5990784946964705, 1.7383903346505918, 1.5307661661772007, 1.4323492558141884, 1.2429043804879258, 1.3172537017266024, 1.2162602046192026, 1.0552102311500136, 0.9826538220680813, 1.0107614019133901, 0.8826720328909949, 0.739013117314438, 0.7006180982562433, 0.737790003472691, 0.6192970376209981, 0.5556544994109884, 0.47239582046029627, 0.4740970917174548, 0.4544292159121539, 0.39665333426527116, 0.35034805233110755, 0.3379110425614577, 0.30972196454961737, 0.31015209244222897, 0.26910402420943297 ], "peak_layer": 5, "peak_normalized": 1.9142257050168354, "population_cv": 0.5160505486771831, "rest_mean": 4.132475101801834e-05, "spike_contrast": 0.5298373517131695, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1895396639592945e-05, "top_five_layers": [ 5, 6, 9, 7, 8 ], "top_five_spike_overlap": 0 }, "values": [ 6.009358912706375e-05, 5.873196278116666e-05, 5.658858208335005e-05, 5.2678646170534194e-05, 7.329362415475771e-05, 6.733737245667726e-05, 6.258225766941905e-05, 6.122697959654033e-05, 6.656107871094719e-05, 5.861137469764799e-05, 5.484309804160148e-05, 4.758945942739956e-05, 5.04362142237369e-05, 4.6569282858399674e-05, 4.0402854210697114e-05, 3.76247480744496e-05, 3.870095679303631e-05, 3.379655390745029e-05, 2.8296009986661375e-05, 2.682590638869442e-05, 2.8249178285477683e-05, 2.3712211259407923e-05, 2.1275407561915927e-05, 1.8087523130816408e-05, 1.8152662960346788e-05, 1.7399601347278804e-05, 1.5187425560725387e-05, 1.3414446584647521e-05, 1.293824698223034e-05, 1.1858917787321843e-05, 1.1875386917381547e-05, 1.0303700946678873e-05 ] } } } } } }, "step": 100 }, { "modes": { "learned": { "bits_per_byte": 3.0609520355704922, "forward": { "activation_sha256": { "attention_branch_output": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", "mlp_branch_output": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", "post_attention_state": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", "post_mlp_state": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", "pre_attention_input": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", "pre_mlp_input": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d" }, "logits_sha256": "b2134e9ad4fc330ab7158864584c7289a039b641b1626767dcba8e1a49e62a3d", "loss_nats": 2.121690273284912, "mixer_summary_sha256": "24302def7a1bb63aa3c7a0b6a721bf1efbd647790e9d6d8427706d94cca54b03" }, "loss_nats": 2.121690273284912, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.658259391784668, "group": 1, "latest_source_mass": 0.6112743616104126, "layer": 1, "max_source_mass": 0.6112743616104126, "mixer_index": 1, "normalized_entropy": 0.9496675601463258, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3887256383895874, "median": 0.36817848682403564, "p05": 0.2958398759365082, "p95": 0.519643247127533 }, { "label": "current_group_partial", "mean": 0.6112743616104126, "median": 0.6318215131759644, "p05": 0.48035678267478943, "p95": 0.7041600942611694 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6694332361221313, "group": 1, "latest_source_mass": 0.5721787214279175, "layer": 2, "max_source_mass": 0.5721787214279175, "mixer_index": 2, "normalized_entropy": 0.9657880099596494, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.42782124876976013, "median": 0.40344932675361633, "p05": 0.3339918255805969, "p95": 0.5918412208557129 }, { "label": "current_group_partial", "mean": 0.5721787214279175, "median": 0.596550703048706, "p05": 0.4081588089466095, "p95": 0.6660082340240479 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6890689730644226, "group": 1, "latest_source_mass": 0.4954703748226166, "layer": 2, "max_source_mass": 0.5045296549797058, "mixer_index": 3, "normalized_entropy": 0.9941163902704933, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5045296549797058, "median": 0.50140380859375, "p05": 0.4251278042793274, "p95": 0.583143949508667 }, { "label": "current_group_partial", "mean": 0.4954703748226166, "median": 0.49859619140625, "p05": 0.41685599088668823, "p95": 0.5748721957206726 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6740086078643799, "group": 1, "latest_source_mass": 0.5477453470230103, "layer": 3, "max_source_mass": 0.5477453470230103, "mixer_index": 4, "normalized_entropy": 0.9723888760824149, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.45225459337234497, "median": 0.42421072721481323, "p05": 0.3554007411003113, "p95": 0.6163875460624695 }, { "label": "current_group_partial", "mean": 0.5477453470230103, "median": 0.575789213180542, "p05": 0.3836124539375305, "p95": 0.644599199295044 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.679557204246521, "group": 1, "latest_source_mass": 0.5538821220397949, "layer": 3, "max_source_mass": 0.5538821220397949, "mixer_index": 5, "normalized_entropy": 0.9803938085668243, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4461178183555603, "median": 0.43929627537727356, "p05": 0.3631950318813324, "p95": 0.562574565410614 }, { "label": "current_group_partial", "mean": 0.5538821220397949, "median": 0.5607037544250488, "p05": 0.4374254047870636, "p95": 0.63680499792099 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6740646362304688, "group": 1, "latest_source_mass": 0.5581084489822388, "layer": 4, "max_source_mass": 0.5581084489822388, "mixer_index": 6, "normalized_entropy": 0.9724697079283204, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.44189155101776123, "median": 0.42476987838745117, "p05": 0.33982762694358826, "p95": 0.5899009108543396 }, { "label": "current_group_partial", "mean": 0.5581084489822388, "median": 0.5752301216125488, "p05": 0.410099059343338, "p95": 0.6601723432540894 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6792672276496887, "group": 1, "latest_source_mass": 0.5544062852859497, "layer": 4, "max_source_mass": 0.5544062852859497, "mixer_index": 7, "normalized_entropy": 0.9799754607686005, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4455936849117279, "median": 0.44227513670921326, "p05": 0.348201721906662, "p95": 0.5532940626144409 }, { "label": "current_group_partial", "mean": 0.5544062852859497, "median": 0.5577248334884644, "p05": 0.4467058777809143, "p95": 0.6517983078956604 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674970269203186, "group": 2, "latest_source_mass": 0.5532590746879578, "layer": 5, "max_source_mass": 0.5532590746879578, "mixer_index": 8, "normalized_entropy": 0.9737762601269252, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.44674086570739746, "median": 0.43155425786972046, "p05": 0.3413114547729492, "p95": 0.5883275270462036 }, { "label": "completed_group_1", "mean": 0.5532590746879578, "median": 0.5684457421302795, "p05": 0.4116724729537964, "p95": 0.6586886048316956 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.093062162399292, "group": 2, "latest_source_mass": 0.35190510749816895, "layer": 5, "max_source_mass": 0.35190510749816895, "mixer_index": 9, "normalized_entropy": 0.99494805735739, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3082374334335327, "median": 0.30379241704940796, "p05": 0.26716136932373047, "p95": 0.3615886867046356 }, { "label": "completed_group_1", "mean": 0.33985745906829834, "median": 0.34154799580574036, "p05": 0.2781605124473572, "p95": 0.3903803527355194 }, { "label": "current_group_partial", "mean": 0.35190510749816895, "median": 0.35370177030563354, "p05": 0.3136623501777649, "p95": 0.3873046040534973 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0853323936462402, "group": 2, "latest_source_mass": 0.37136080861091614, "layer": 6, "max_source_mass": 0.37136080861091614, "mixer_index": 10, "normalized_entropy": 0.9879121186256079, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30180174112319946, "median": 0.28923720121383667, "p05": 0.2442891150712967, "p95": 0.3929156959056854 }, { "label": "completed_group_1", "mean": 0.3268374800682068, "median": 0.3262294828891754, "p05": 0.2930828928947449, "p95": 0.3670434355735779 }, { "label": "current_group_partial", "mean": 0.37136080861091614, "median": 0.38435930013656616, "p05": 0.2483101636171341, "p95": 0.4514022171497345 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0921990871429443, "group": 2, "latest_source_mass": 0.3605259656906128, "layer": 6, "max_source_mass": 0.3605259656906128, "mixer_index": 11, "normalized_entropy": 0.9941624524035314, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3034760355949402, "median": 0.30043965578079224, "p05": 0.2575032711029053, "p95": 0.3658854067325592 }, { "label": "completed_group_1", "mean": 0.3359980285167694, "median": 0.33590996265411377, "p05": 0.29249265789985657, "p95": 0.3839111328125 }, { "label": "current_group_partial", "mean": 0.3605259656906128, "median": 0.3592720031738281, "p05": 0.3142351508140564, "p95": 0.40569764375686646 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.076724648475647, "group": 2, "latest_source_mass": 0.3958718776702881, "layer": 7, "max_source_mass": 0.3958718776702881, "mixer_index": 12, "normalized_entropy": 0.9800770113185262, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29931002855300903, "median": 0.2845022678375244, "p05": 0.24265936017036438, "p95": 0.3946967124938965 }, { "label": "completed_group_1", "mean": 0.30481812357902527, "median": 0.2940869927406311, "p05": 0.26745280623435974, "p95": 0.36952510476112366 }, { "label": "current_group_partial", "mean": 0.3958718776702881, "median": 0.4253297448158264, "p05": 0.24267785251140594, "p95": 0.4795311689376831 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0856893062591553, "group": 2, "latest_source_mass": 0.3888435363769531, "layer": 7, "max_source_mass": 0.3888435363769531, "mixer_index": 13, "normalized_entropy": 0.988236994486361, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2801036834716797, "median": 0.2685231566429138, "p05": 0.24713139235973358, "p95": 0.3503561317920685 }, { "label": "completed_group_1", "mean": 0.3310527801513672, "median": 0.330751895904541, "p05": 0.2913045883178711, "p95": 0.37040621042251587 }, { "label": "current_group_partial", "mean": 0.3888435363769531, "median": 0.3877270221710205, "p05": 0.3418702185153961, "p95": 0.44118839502334595 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0752873420715332, "group": 2, "latest_source_mass": 0.3948463201522827, "layer": 8, "max_source_mass": 0.3948463201522827, "mixer_index": 14, "normalized_entropy": 0.9787687186488199, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29463040828704834, "median": 0.28890830278396606, "p05": 0.23444202542304993, "p95": 0.37455716729164124 }, { "label": "completed_group_1", "mean": 0.3105233311653137, "median": 0.2965858280658722, "p05": 0.2587319016456604, "p95": 0.39421546459198 }, { "label": "current_group_partial", "mean": 0.3948463201522827, "median": 0.4135509133338928, "p05": 0.24363072216510773, "p95": 0.5001816153526306 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.088913917541504, "group": 2, "latest_source_mass": 0.3843393623828888, "layer": 8, "max_source_mass": 0.3843393623828888, "mixer_index": 15, "normalized_entropy": 0.9911721621661782, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29400017857551575, "median": 0.2848791778087616, "p05": 0.2631606459617615, "p95": 0.35448360443115234 }, { "label": "completed_group_1", "mean": 0.32166045904159546, "median": 0.32034605741500854, "p05": 0.2860022187232971, "p95": 0.3604046106338501 }, { "label": "current_group_partial", "mean": 0.3843393623828888, "median": 0.38500386476516724, "p05": 0.336969792842865, "p95": 0.43113112449645996 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.054490327835083, "group": 3, "latest_source_mass": 0.42586958408355713, "layer": 9, "max_source_mass": 0.42586958408355713, "mixer_index": 16, "normalized_entropy": 0.959838460494086, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2850497364997864, "median": 0.2810911536216736, "p05": 0.21189084649085999, "p95": 0.37678587436676025 }, { "label": "completed_group_1", "mean": 0.2890806794166565, "median": 0.26693958044052124, "p05": 0.21384978294372559, "p95": 0.4104633927345276 }, { "label": "completed_group_2", "mean": 0.42586958408355713, "median": 0.45180952548980713, "p05": 0.2289215475320816, "p95": 0.5656958818435669 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3700100183486938, "group": 3, "latest_source_mass": 0.28284209966659546, "layer": 9, "max_source_mass": 0.28284209966659546, "mixer_index": 17, "normalized_entropy": 0.9882533297199292, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21610885858535767, "median": 0.21061542630195618, "p05": 0.18654318153858185, "p95": 0.26685822010040283 }, { "label": "completed_group_1", "mean": 0.22824729979038239, "median": 0.22523541748523712, "p05": 0.19356240332126617, "p95": 0.2682129740715027 }, { "label": "completed_group_2", "mean": 0.2728016972541809, "median": 0.27193963527679443, "p05": 0.21500371396541595, "p95": 0.3283920884132385 }, { "label": "current_group_partial", "mean": 0.28284209966659546, "median": 0.2946581244468689, "p05": 0.19060052931308746, "p95": 0.3525015413761139 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3620216846466064, "group": 3, "latest_source_mass": 0.2944645285606384, "layer": 10, "max_source_mass": 0.2944645285606384, "mixer_index": 18, "normalized_entropy": 0.9824909650114454, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.22791308164596558, "median": 0.22349298000335693, "p05": 0.18910717964172363, "p95": 0.2777109742164612 }, { "label": "completed_group_1", "mean": 0.2214205265045166, "median": 0.19864881038665771, "p05": 0.15984180569648743, "p95": 0.34694117307662964 }, { "label": "completed_group_2", "mean": 0.2562018632888794, "median": 0.2503240704536438, "p05": 0.20511329174041748, "p95": 0.32819920778274536 }, { "label": "current_group_partial", "mean": 0.2944645285606384, "median": 0.30793535709381104, "p05": 0.16727104783058167, "p95": 0.3644053339958191 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3784711360931396, "group": 3, "latest_source_mass": 0.2763957977294922, "layer": 10, "max_source_mass": 0.2763957977294922, "mixer_index": 19, "normalized_entropy": 0.994356736025074, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.23347607254981995, "median": 0.23056401312351227, "p05": 0.20095914602279663, "p95": 0.2848108112812042 }, { "label": "completed_group_1", "mean": 0.23307976126670837, "median": 0.2289559543132782, "p05": 0.205013245344162, "p95": 0.2760506868362427 }, { "label": "completed_group_2", "mean": 0.2570483684539795, "median": 0.2558031678199768, "p05": 0.21930555999279022, "p95": 0.29454779624938965 }, { "label": "current_group_partial", "mean": 0.2763957977294922, "median": 0.2743210792541504, "p05": 0.2263108193874359, "p95": 0.3325941264629364 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3619427680969238, "group": 3, "latest_source_mass": 0.302674800157547, "layer": 11, "max_source_mass": 0.302674800157547, "mixer_index": 20, "normalized_entropy": 0.9824340387540098, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.24715131521224976, "median": 0.24610307812690735, "p05": 0.21518638730049133, "p95": 0.2835894525051117 }, { "label": "completed_group_1", "mean": 0.22003203630447388, "median": 0.19626209139823914, "p05": 0.16581544280052185, "p95": 0.3317301273345947 }, { "label": "completed_group_2", "mean": 0.23014184832572937, "median": 0.2278168797492981, "p05": 0.19933417439460754, "p95": 0.272026926279068 }, { "label": "current_group_partial", "mean": 0.302674800157547, "median": 0.3237387239933014, "p05": 0.16210556030273438, "p95": 0.38430726528167725 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3804662227630615, "group": 3, "latest_source_mass": 0.28360050916671753, "layer": 11, "max_source_mass": 0.28360050916671753, "mixer_index": 21, "normalized_entropy": 0.995795886847494, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.24302780628204346, "median": 0.24201302230358124, "p05": 0.21055130660533905, "p95": 0.276764452457428 }, { "label": "completed_group_1", "mean": 0.23108476400375366, "median": 0.22807812690734863, "p05": 0.20644722878932953, "p95": 0.2614594101905823 }, { "label": "completed_group_2", "mean": 0.24228686094284058, "median": 0.2414286881685257, "p05": 0.21957996487617493, "p95": 0.2676010727882385 }, { "label": "current_group_partial", "mean": 0.28360050916671753, "median": 0.2854132354259491, "p05": 0.246682807803154, "p95": 0.3173881471157074 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3502743244171143, "group": 3, "latest_source_mass": 0.31918564438819885, "layer": 12, "max_source_mass": 0.31918564438819885, "mixer_index": 22, "normalized_entropy": 0.974017035838133, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2455797642469406, "median": 0.2419765591621399, "p05": 0.2023906707763672, "p95": 0.29628193378448486 }, { "label": "completed_group_1", "mean": 0.21441738307476044, "median": 0.19096502661705017, "p05": 0.1489589512348175, "p95": 0.3387751877307892 }, { "label": "completed_group_2", "mean": 0.2208172082901001, "median": 0.21868902444839478, "p05": 0.19425739347934723, "p95": 0.25476256012916565 }, { "label": "current_group_partial", "mean": 0.31918564438819885, "median": 0.34166574478149414, "p05": 0.153193861246109, "p95": 0.42244112491607666 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3820215463638306, "group": 3, "latest_source_mass": 0.27603140473365784, "layer": 12, "max_source_mass": 0.27603140473365784, "mixer_index": 23, "normalized_entropy": 0.9969178156703975, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.24522465467453003, "median": 0.24368876218795776, "p05": 0.22978872060775757, "p95": 0.2647436261177063 }, { "label": "completed_group_1", "mean": 0.22846508026123047, "median": 0.22824203968048096, "p05": 0.21080705523490906, "p95": 0.24792201817035675 }, { "label": "completed_group_2", "mean": 0.25027889013290405, "median": 0.25114861130714417, "p05": 0.2239076942205429, "p95": 0.27430635690689087 }, { "label": "current_group_partial", "mean": 0.27603140473365784, "median": 0.2767183184623718, "p05": 0.23810210824012756, "p95": 0.31143108010292053 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3491160869598389, "group": 4, "latest_source_mass": 0.3227139413356781, "layer": 13, "max_source_mass": 0.3227139413356781, "mixer_index": 24, "normalized_entropy": 0.9731815441202416, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24498310685157776, "median": 0.24184581637382507, "p05": 0.20811651647090912, "p95": 0.2852743864059448 }, { "label": "completed_group_1", "mean": 0.21178749203681946, "median": 0.1895410120487213, "p05": 0.14949846267700195, "p95": 0.3285062313079834 }, { "label": "completed_group_2", "mean": 0.2205154001712799, "median": 0.2162817418575287, "p05": 0.18882519006729126, "p95": 0.26559314131736755 }, { "label": "completed_group_3", "mean": 0.3227139413356781, "median": 0.34892457723617554, "p05": 0.15726026892662048, "p95": 0.42967018485069275 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6032840013504028, "group": 4, "latest_source_mass": 0.22103822231292725, "layer": 13, "max_source_mass": 0.22103822231292725, "mixer_index": 25, "normalized_entropy": 0.9961763600595251, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20074918866157532, "median": 0.20095838606357574, "p05": 0.18337497115135193, "p95": 0.2181507796049118 }, { "label": "completed_group_1", "mean": 0.17959120869636536, "median": 0.17876555025577545, "p05": 0.1576874852180481, "p95": 0.20805387198925018 }, { "label": "completed_group_2", "mean": 0.18619459867477417, "median": 0.18535691499710083, "p05": 0.1644289195537567, "p95": 0.21231110394001007 }, { "label": "completed_group_3", "mean": 0.21242676675319672, "median": 0.2123909741640091, "p05": 0.19269250333309174, "p95": 0.23113366961479187 }, { "label": "current_group_partial", "mean": 0.22103822231292725, "median": 0.2211262434720993, "p05": 0.1813979148864746, "p95": 0.26087528467178345 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5699318647384644, "group": 4, "latest_source_mass": 0.2694075107574463, "layer": 14, "max_source_mass": 0.2694075107574463, "mixer_index": 26, "normalized_entropy": 0.9754535124403232, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1938496232032776, "median": 0.19102387130260468, "p05": 0.16569040715694427, "p95": 0.23268038034439087 }, { "label": "completed_group_1", "mean": 0.17019829154014587, "median": 0.14846302568912506, "p05": 0.11795832216739655, "p95": 0.2854715883731842 }, { "label": "completed_group_2", "mean": 0.1625727415084839, "median": 0.15400215983390808, "p05": 0.1330171525478363, "p95": 0.22402943670749664 }, { "label": "completed_group_3", "mean": 0.20397180318832397, "median": 0.2051275074481964, "p05": 0.14910343289375305, "p95": 0.25531816482543945 }, { "label": "current_group_partial", "mean": 0.2694075107574463, "median": 0.2924001216888428, "p05": 0.12448020279407501, "p95": 0.3373839557170868 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6068798303604126, "group": 4, "latest_source_mass": 0.21404394507408142, "layer": 14, "max_source_mass": 0.21404394507408142, "mixer_index": 27, "normalized_entropy": 0.9984105742421472, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19717517495155334, "median": 0.19712674617767334, "p05": 0.18285326659679413, "p95": 0.21078822016716003 }, { "label": "completed_group_1", "mean": 0.1942940056324005, "median": 0.19466334581375122, "p05": 0.18043261766433716, "p95": 0.20777744054794312 }, { "label": "completed_group_2", "mean": 0.19104912877082825, "median": 0.18969203531742096, "p05": 0.17138653993606567, "p95": 0.2142438441514969 }, { "label": "completed_group_3", "mean": 0.20343773066997528, "median": 0.2029801905155182, "p05": 0.18992920219898224, "p95": 0.2184751033782959 }, { "label": "current_group_partial", "mean": 0.21404394507408142, "median": 0.21432773768901825, "p05": 0.1852298229932785, "p95": 0.24286989867687225 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5761134624481201, "group": 4, "latest_source_mass": 0.26560431718826294, "layer": 15, "max_source_mass": 0.26560431718826294, "mixer_index": 28, "normalized_entropy": 0.979294355048726, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1940101534128189, "median": 0.19362978637218475, "p05": 0.1631820648908615, "p95": 0.2295525223016739 }, { "label": "completed_group_1", "mean": 0.18250693380832672, "median": 0.16429191827774048, "p05": 0.13507941365242004, "p95": 0.2775976359844208 }, { "label": "completed_group_2", "mean": 0.16849923133850098, "median": 0.16119162738323212, "p05": 0.1375115066766739, "p95": 0.2266949564218521 }, { "label": "completed_group_3", "mean": 0.18937936425209045, "median": 0.18853023648262024, "p05": 0.1678447425365448, "p95": 0.21501585841178894 }, { "label": "current_group_partial", "mean": 0.26560431718826294, "median": 0.28881219029426575, "p05": 0.117766834795475, "p95": 0.34617751836776733 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6027262210845947, "group": 4, "latest_source_mass": 0.22770702838897705, "layer": 15, "max_source_mass": 0.22770702838897705, "mixer_index": 29, "normalized_entropy": 0.9958297916945706, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20452985167503357, "median": 0.20441509783267975, "p05": 0.17931760847568512, "p95": 0.2289508432149887 }, { "label": "completed_group_1", "mean": 0.18702909350395203, "median": 0.186691552400589, "p05": 0.1675943285226822, "p95": 0.20997506380081177 }, { "label": "completed_group_2", "mean": 0.17977532744407654, "median": 0.177423357963562, "p05": 0.1569271981716156, "p95": 0.20937731862068176 }, { "label": "completed_group_3", "mean": 0.20095868408679962, "median": 0.2005838304758072, "p05": 0.1845097541809082, "p95": 0.21870870888233185 }, { "label": "current_group_partial", "mean": 0.22770702838897705, "median": 0.22757767140865326, "p05": 0.18680104613304138, "p95": 0.2673209607601166 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.577937126159668, "group": 4, "latest_source_mass": 0.2687118649482727, "layer": 16, "max_source_mass": 0.2687118649482727, "mixer_index": 30, "normalized_entropy": 0.9804274610215993, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1962222456932068, "median": 0.19565686583518982, "p05": 0.16995516419410706, "p95": 0.22565382719039917 }, { "label": "completed_group_1", "mean": 0.1818992793560028, "median": 0.16701740026474, "p05": 0.138178750872612, "p95": 0.2653764486312866 }, { "label": "completed_group_2", "mean": 0.17167484760284424, "median": 0.16360890865325928, "p05": 0.14424335956573486, "p95": 0.22631293535232544 }, { "label": "completed_group_3", "mean": 0.18149182200431824, "median": 0.18099229037761688, "p05": 0.16264864802360535, "p95": 0.20300358533859253 }, { "label": "current_group_partial", "mean": 0.2687118649482727, "median": 0.29608893394470215, "p05": 0.1254507452249527, "p95": 0.339657723903656 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5977532863616943, "group": 4, "latest_source_mass": 0.23852142691612244, "layer": 16, "max_source_mass": 0.23852142691612244, "mixer_index": 31, "normalized_entropy": 0.9927399336239481, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21533392369747162, "median": 0.21377520263195038, "p05": 0.1841924786567688, "p95": 0.24801591038703918 }, { "label": "completed_group_1", "mean": 0.18402470648288727, "median": 0.18508511781692505, "p05": 0.15855011343955994, "p95": 0.21014195680618286 }, { "label": "completed_group_2", "mean": 0.17019468545913696, "median": 0.16691477596759796, "p05": 0.1446647047996521, "p95": 0.20511509478092194 }, { "label": "completed_group_3", "mean": 0.1919252574443817, "median": 0.191789448261261, "p05": 0.17424410581588745, "p95": 0.20975348353385925 }, { "label": "current_group_partial", "mean": 0.23852142691612244, "median": 0.24018850922584534, "p05": 0.18989041447639465, "p95": 0.27930569648742676 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.578121542930603, "group": 5, "latest_source_mass": 0.26854145526885986, "layer": 17, "max_source_mass": 0.26854145526885986, "mixer_index": 32, "normalized_entropy": 0.9805420456038999, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20422279834747314, "median": 0.2033933699131012, "p05": 0.17994040250778198, "p95": 0.23156529664993286 }, { "label": "completed_group_1", "mean": 0.1835813671350479, "median": 0.17072421312332153, "p05": 0.14088554680347443, "p95": 0.26194027066230774 }, { "label": "completed_group_2", "mean": 0.16642534732818604, "median": 0.1602073311805725, "p05": 0.13873112201690674, "p95": 0.22039896249771118 }, { "label": "completed_group_3", "mean": 0.17722904682159424, "median": 0.17722415924072266, "p05": 0.1577582061290741, "p95": 0.19813896715641022 }, { "label": "completed_group_4", "mean": 0.26854145526885986, "median": 0.2872956693172455, "p05": 0.12619024515151978, "p95": 0.3464823067188263 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.782798409461975, "group": 5, "latest_source_mass": 0.19473549723625183, "layer": 17, "max_source_mass": 0.19473549723625183, "mixer_index": 33, "normalized_entropy": 0.99499873731939, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15961387753486633, "median": 0.15747272968292236, "p05": 0.12699076533317566, "p95": 0.19599634408950806 }, { "label": "completed_group_1", "mean": 0.16475997865200043, "median": 0.16349202394485474, "p05": 0.1474122852087021, "p95": 0.1848660409450531 }, { "label": "completed_group_2", "mean": 0.14714640378952026, "median": 0.14494061470031738, "p05": 0.12754470109939575, "p95": 0.1751968413591385 }, { "label": "completed_group_3", "mean": 0.15768736600875854, "median": 0.15663693845272064, "p05": 0.13646987080574036, "p95": 0.1821092814207077 }, { "label": "completed_group_4", "mean": 0.1760568618774414, "median": 0.17550913989543915, "p05": 0.15776243805885315, "p95": 0.19570523500442505 }, { "label": "current_group_partial", "mean": 0.19473549723625183, "median": 0.1956680715084076, "p05": 0.14822278916835785, "p95": 0.22800904512405396 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7707297801971436, "group": 5, "latest_source_mass": 0.20674791932106018, "layer": 18, "max_source_mass": 0.20674791932106018, "mixer_index": 34, "normalized_entropy": 0.9882631070787802, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16306114196777344, "median": 0.16283683478832245, "p05": 0.13633866608142853, "p95": 0.19143974781036377 }, { "label": "completed_group_1", "mean": 0.1578725278377533, "median": 0.14938931167125702, "p05": 0.13056670129299164, "p95": 0.21463912725448608 }, { "label": "completed_group_2", "mean": 0.139655202627182, "median": 0.1338472068309784, "p05": 0.11858384311199188, "p95": 0.18817788362503052 }, { "label": "completed_group_3", "mean": 0.14299750328063965, "median": 0.14077699184417725, "p05": 0.12273441255092621, "p95": 0.17249858379364014 }, { "label": "completed_group_4", "mean": 0.18966571986675262, "median": 0.19467708468437195, "p05": 0.13032010197639465, "p95": 0.23041488230228424 }, { "label": "current_group_partial", "mean": 0.20674791932106018, "median": 0.21543893218040466, "p05": 0.11787950992584229, "p95": 0.24730797111988068 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.781312108039856, "group": 5, "latest_source_mass": 0.20407646894454956, "layer": 18, "max_source_mass": 0.20407646894454956, "mixer_index": 35, "normalized_entropy": 0.9941692167014471, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.15919369459152222, "median": 0.15759795904159546, "p05": 0.13659372925758362, "p95": 0.19095730781555176 }, { "label": "completed_group_1", "mean": 0.1684478521347046, "median": 0.16807237267494202, "p05": 0.15075388550758362, "p95": 0.1875023990869522 }, { "label": "completed_group_2", "mean": 0.14742717146873474, "median": 0.14584535360336304, "p05": 0.12703484296798706, "p95": 0.1739335060119629 }, { "label": "completed_group_3", "mean": 0.1539241522550583, "median": 0.1541770100593567, "p05": 0.13218382000923157, "p95": 0.1755600869655609 }, { "label": "completed_group_4", "mean": 0.1669306606054306, "median": 0.16647151112556458, "p05": 0.15041440725326538, "p95": 0.18558931350708008 }, { "label": "current_group_partial", "mean": 0.20407646894454956, "median": 0.20521500706672668, "p05": 0.1562502235174179, "p95": 0.24481025338172913 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7736209630966187, "group": 5, "latest_source_mass": 0.2125461846590042, "layer": 19, "max_source_mass": 0.2125461846590042, "mixer_index": 36, "normalized_entropy": 0.9898767069782805, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15695160627365112, "median": 0.15683278441429138, "p05": 0.12300770729780197, "p95": 0.196682408452034 }, { "label": "completed_group_1", "mean": 0.16567477583885193, "median": 0.16040010750293732, "p05": 0.14332951605319977, "p95": 0.20587366819381714 }, { "label": "completed_group_2", "mean": 0.1478763222694397, "median": 0.14480885863304138, "p05": 0.12841497361660004, "p95": 0.18284933269023895 }, { "label": "completed_group_3", "mean": 0.14320014417171478, "median": 0.14095112681388855, "p05": 0.12106460332870483, "p95": 0.17675133049488068 }, { "label": "completed_group_4", "mean": 0.17375096678733826, "median": 0.17515000700950623, "p05": 0.1435358226299286, "p95": 0.19906878471374512 }, { "label": "current_group_partial", "mean": 0.2125461846590042, "median": 0.2186175137758255, "p05": 0.12126382440328598, "p95": 0.2633163630962372 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.781569480895996, "group": 5, "latest_source_mass": 0.20039017498493195, "layer": 19, "max_source_mass": 0.20039017498493195, "mixer_index": 37, "normalized_entropy": 0.9943128592274447, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15368077158927917, "median": 0.1525704562664032, "p05": 0.12359238415956497, "p95": 0.20425409078598022 }, { "label": "completed_group_1", "mean": 0.1725173443555832, "median": 0.17324355244636536, "p05": 0.1533699631690979, "p95": 0.1897408664226532 }, { "label": "completed_group_2", "mean": 0.14978237450122833, "median": 0.14767895638942719, "p05": 0.1299348920583725, "p95": 0.17630663514137268 }, { "label": "completed_group_3", "mean": 0.15674246847629547, "median": 0.15763753652572632, "p05": 0.13473035395145416, "p95": 0.17680563032627106 }, { "label": "completed_group_4", "mean": 0.16688686609268188, "median": 0.16692788898944855, "p05": 0.14797358214855194, "p95": 0.18507987260818481 }, { "label": "current_group_partial", "mean": 0.20039017498493195, "median": 0.20044702291488647, "p05": 0.1619955599308014, "p95": 0.23957164585590363 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7793617248535156, "group": 5, "latest_source_mass": 0.20417512953281403, "layer": 20, "max_source_mass": 0.20417512953281403, "mixer_index": 38, "normalized_entropy": 0.9930806871193036, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1678246557712555, "median": 0.17157375812530518, "p05": 0.13025718927383423, "p95": 0.19556939601898193 }, { "label": "completed_group_1", "mean": 0.1683025360107422, "median": 0.16609728336334229, "p05": 0.15072931349277496, "p95": 0.19358061254024506 }, { "label": "completed_group_2", "mean": 0.14878366887569427, "median": 0.1468905806541443, "p05": 0.13322587311267853, "p95": 0.1712985783815384 }, { "label": "completed_group_3", "mean": 0.14351718127727509, "median": 0.1410684883594513, "p05": 0.12425544857978821, "p95": 0.1736954152584076 }, { "label": "completed_group_4", "mean": 0.16739682853221893, "median": 0.16813713312149048, "p05": 0.14868372678756714, "p95": 0.1844603419303894 }, { "label": "current_group_partial", "mean": 0.20417512953281403, "median": 0.20674002170562744, "p05": 0.14016404747962952, "p95": 0.2501453757286072 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7772380113601685, "group": 5, "latest_source_mass": 0.19795814156532288, "layer": 20, "max_source_mass": 0.19795814156532288, "mixer_index": 39, "normalized_entropy": 0.9918954200509164, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15367703139781952, "median": 0.15482746064662933, "p05": 0.11167150735855103, "p95": 0.2207922786474228 }, { "label": "completed_group_1", "mean": 0.18459689617156982, "median": 0.184708833694458, "p05": 0.160706028342247, "p95": 0.20910656452178955 }, { "label": "completed_group_2", "mean": 0.14832255244255066, "median": 0.1467352658510208, "p05": 0.12785059213638306, "p95": 0.17438356578350067 }, { "label": "completed_group_3", "mean": 0.15016385912895203, "median": 0.14991271495819092, "p05": 0.12576226890087128, "p95": 0.17565840482711792 }, { "label": "completed_group_4", "mean": 0.1652815043926239, "median": 0.16595271229743958, "p05": 0.14006288349628448, "p95": 0.18747010827064514 }, { "label": "current_group_partial", "mean": 0.19795814156532288, "median": 0.19790686666965485, "p05": 0.1500549614429474, "p95": 0.24641191959381104 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.780207633972168, "group": 6, "latest_source_mass": 0.20078188180923462, "layer": 21, "max_source_mass": 0.20078188180923462, "mixer_index": 40, "normalized_entropy": 0.9935527979875202, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16951784491539001, "median": 0.1744978278875351, "p05": 0.12857238948345184, "p95": 0.2016112506389618 }, { "label": "completed_group_1", "mean": 0.17200809717178345, "median": 0.17267316579818726, "p05": 0.15072743594646454, "p95": 0.19161207973957062 }, { "label": "completed_group_2", "mean": 0.15038041770458221, "median": 0.14867597818374634, "p05": 0.13418853282928467, "p95": 0.17207789421081543 }, { "label": "completed_group_3", "mean": 0.14669056236743927, "median": 0.14433687925338745, "p05": 0.12725189328193665, "p95": 0.1738971769809723 }, { "label": "completed_group_4", "mean": 0.16062124073505402, "median": 0.16014984250068665, "p05": 0.14366847276687622, "p95": 0.17978575825691223 }, { "label": "completed_group_5", "mean": 0.20078188180923462, "median": 0.20047783851623535, "p05": 0.14466112852096558, "p95": 0.25333014130592346 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.937079668045044, "group": 6, "latest_source_mass": 0.16695083677768707, "layer": 21, "max_source_mass": 0.16695083677768707, "mixer_index": 41, "normalized_entropy": 0.9954620304464951, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13008031249046326, "median": 0.12692952156066895, "p05": 0.10487981885671616, "p95": 0.16793549060821533 }, { "label": "completed_group_1", "mean": 0.15163564682006836, "median": 0.15097364783287048, "p05": 0.13223227858543396, "p95": 0.17222966253757477 }, { "label": "completed_group_2", "mean": 0.1323973685503006, "median": 0.131024569272995, "p05": 0.11479418724775314, "p95": 0.15471340715885162 }, { "label": "completed_group_3", "mean": 0.13641127943992615, "median": 0.13619393110275269, "p05": 0.11877292394638062, "p95": 0.15510697662830353 }, { "label": "completed_group_4", "mean": 0.13806137442588806, "median": 0.1375710368156433, "p05": 0.11978909373283386, "p95": 0.15855638682842255 }, { "label": "completed_group_5", "mean": 0.1444631814956665, "median": 0.1438722014427185, "p05": 0.12040875852108002, "p95": 0.17076769471168518 }, { "label": "current_group_partial", "mean": 0.16695083677768707, "median": 0.17087092995643616, "p05": 0.13015522062778473, "p95": 0.195466548204422 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.939245343208313, "group": 6, "latest_source_mass": 0.16357660293579102, "layer": 22, "max_source_mass": 0.16357660293579102, "mixer_index": 42, "normalized_entropy": 0.9965749673230103, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14692580699920654, "median": 0.1518530696630478, "p05": 0.11718790233135223, "p95": 0.16568630933761597 }, { "label": "completed_group_1", "mean": 0.14671877026557922, "median": 0.14763988554477692, "p05": 0.12195808440446854, "p95": 0.1661134660243988 }, { "label": "completed_group_2", "mean": 0.13336658477783203, "median": 0.13267168402671814, "p05": 0.12229838222265244, "p95": 0.14625124633312225 }, { "label": "completed_group_3", "mean": 0.1307726800441742, "median": 0.12948819994926453, "p05": 0.11789381504058838, "p95": 0.1479169726371765 }, { "label": "completed_group_4", "mean": 0.131200909614563, "median": 0.1309259533882141, "p05": 0.11869271099567413, "p95": 0.14432916045188904 }, { "label": "completed_group_5", "mean": 0.147438645362854, "median": 0.14618173241615295, "p05": 0.13021595776081085, "p95": 0.16755904257297516 }, { "label": "current_group_partial", "mean": 0.16357660293579102, "median": 0.1623307466506958, "p05": 0.13464656472206116, "p95": 0.1969548761844635 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9312366247177124, "group": 6, "latest_source_mass": 0.1666574329137802, "layer": 22, "max_source_mass": 0.16950437426567078, "mixer_index": 43, "normalized_entropy": 0.9924593001661848, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12334204465150833, "median": 0.12106236815452576, "p05": 0.08947738260030746, "p95": 0.17906470596790314 }, { "label": "completed_group_1", "mean": 0.16950437426567078, "median": 0.17181579768657684, "p05": 0.1329171359539032, "p95": 0.19668330252170563 }, { "label": "completed_group_2", "mean": 0.13870283961296082, "median": 0.13818645477294922, "p05": 0.12228117883205414, "p95": 0.1556926816701889 }, { "label": "completed_group_3", "mean": 0.1300538182258606, "median": 0.13022829592227936, "p05": 0.1119839996099472, "p95": 0.14633683860301971 }, { "label": "completed_group_4", "mean": 0.12885478138923645, "median": 0.1284654438495636, "p05": 0.11224459111690521, "p95": 0.14672259986400604 }, { "label": "completed_group_5", "mean": 0.14288470149040222, "median": 0.14101536571979523, "p05": 0.11797811090946198, "p95": 0.17301785945892334 }, { "label": "current_group_partial", "mean": 0.1666574329137802, "median": 0.1637348234653473, "p05": 0.12865713238716125, "p95": 0.20903055369853973 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9345850944519043, "group": 6, "latest_source_mass": 0.16773909330368042, "layer": 23, "max_source_mass": 0.16773909330368042, "mixer_index": 44, "normalized_entropy": 0.9941800732120613, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1465124636888504, "median": 0.14716467261314392, "p05": 0.12261509895324707, "p95": 0.17156414687633514 }, { "label": "completed_group_1", "mean": 0.153631791472435, "median": 0.159563809633255, "p05": 0.10715117305517197, "p95": 0.18623968958854675 }, { "label": "completed_group_2", "mean": 0.13159149885177612, "median": 0.13108083605766296, "p05": 0.11123912036418915, "p95": 0.1546429693698883 }, { "label": "completed_group_3", "mean": 0.13120220601558685, "median": 0.13034585118293762, "p05": 0.11503913998603821, "p95": 0.1493607610464096 }, { "label": "completed_group_4", "mean": 0.12539167702198029, "median": 0.12449093163013458, "p05": 0.10983049124479294, "p95": 0.14342987537384033 }, { "label": "completed_group_5", "mean": 0.14393125474452972, "median": 0.1405339539051056, "p05": 0.1222267672419548, "p95": 0.17515066266059875 }, { "label": "current_group_partial", "mean": 0.16773909330368042, "median": 0.16424956917762756, "p05": 0.13410812616348267, "p95": 0.21529987454414368 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9284589290618896, "group": 6, "latest_source_mass": 0.16532829403877258, "layer": 23, "max_source_mass": 0.169878289103508, "mixer_index": 45, "normalized_entropy": 0.9910318469730498, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12412484735250473, "median": 0.12159939110279083, "p05": 0.08492134511470795, "p95": 0.18361014127731323 }, { "label": "completed_group_1", "mean": 0.169878289103508, "median": 0.17313054203987122, "p05": 0.1298009753227234, "p95": 0.1993248164653778 }, { "label": "completed_group_2", "mean": 0.13807371258735657, "median": 0.13714420795440674, "p05": 0.12058109790086746, "p95": 0.15811002254486084 }, { "label": "completed_group_3", "mean": 0.13034150004386902, "median": 0.13021467626094818, "p05": 0.11215148121118546, "p95": 0.14708518981933594 }, { "label": "completed_group_4", "mean": 0.13028821349143982, "median": 0.13050158321857452, "p05": 0.1127624660730362, "p95": 0.14778561890125275 }, { "label": "completed_group_5", "mean": 0.1419651210308075, "median": 0.1401965618133545, "p05": 0.11490530520677567, "p95": 0.17368361353874207 }, { "label": "current_group_partial", "mean": 0.16532829403877258, "median": 0.16024862229824066, "p05": 0.1169055625796318, "p95": 0.22413846850395203 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9329564571380615, "group": 6, "latest_source_mass": 0.17254900932312012, "layer": 24, "max_source_mass": 0.17254900932312012, "mixer_index": 46, "normalized_entropy": 0.9933431191961559, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1454196274280548, "median": 0.145169198513031, "p05": 0.12175656855106354, "p95": 0.17354263365268707 }, { "label": "completed_group_1", "mean": 0.1566689908504486, "median": 0.16300347447395325, "p05": 0.11168468743562698, "p95": 0.18563294410705566 }, { "label": "completed_group_2", "mean": 0.1286500096321106, "median": 0.12701794505119324, "p05": 0.11111085116863251, "p95": 0.15331649780273438 }, { "label": "completed_group_3", "mean": 0.12666445970535278, "median": 0.12615132331848145, "p05": 0.11176232993602753, "p95": 0.14301280677318573 }, { "label": "completed_group_4", "mean": 0.12782414257526398, "median": 0.1281316727399826, "p05": 0.11212349683046341, "p95": 0.14346082508563995 }, { "label": "completed_group_5", "mean": 0.14222374558448792, "median": 0.14010348916053772, "p05": 0.12181462347507477, "p95": 0.16952435672283173 }, { "label": "current_group_partial", "mean": 0.17254900932312012, "median": 0.1675749123096466, "p05": 0.1353597342967987, "p95": 0.2285366952419281 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9259672164916992, "group": 6, "latest_source_mass": 0.16209754347801208, "layer": 24, "max_source_mass": 0.17791953682899475, "mixer_index": 47, "normalized_entropy": 0.989751360013567, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12082186341285706, "median": 0.11626945436000824, "p05": 0.08509357273578644, "p95": 0.18624860048294067 }, { "label": "completed_group_1", "mean": 0.17791953682899475, "median": 0.1807878017425537, "p05": 0.1341632455587387, "p95": 0.2101013958454132 }, { "label": "completed_group_2", "mean": 0.1424642950296402, "median": 0.14065685868263245, "p05": 0.12273216992616653, "p95": 0.16812381148338318 }, { "label": "completed_group_3", "mean": 0.13298648595809937, "median": 0.13262662291526794, "p05": 0.11336477845907211, "p95": 0.15321806073188782 }, { "label": "completed_group_4", "mean": 0.12929362058639526, "median": 0.12950879335403442, "p05": 0.1104152500629425, "p95": 0.1479472517967224 }, { "label": "completed_group_5", "mean": 0.1344166398048401, "median": 0.13280367851257324, "p05": 0.1057823970913887, "p95": 0.16791269183158875 }, { "label": "current_group_partial", "mean": 0.16209754347801208, "median": 0.15705879032611847, "p05": 0.1103038564324379, "p95": 0.22623871266841888 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9201786518096924, "group": 7, "latest_source_mass": 0.15931037068367004, "layer": 25, "max_source_mass": 0.17355048656463623, "mixer_index": 48, "normalized_entropy": 0.9867766262187836, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13756656646728516, "median": 0.14409756660461426, "p05": 0.07605770230293274, "p95": 0.1807018518447876 }, { "label": "completed_group_1", "mean": 0.17355048656463623, "median": 0.1850908100605011, "p05": 0.09468921273946762, "p95": 0.2254364788532257 }, { "label": "completed_group_2", "mean": 0.14306488633155823, "median": 0.1434939205646515, "p05": 0.12189669907093048, "p95": 0.16250990331172943 }, { "label": "completed_group_3", "mean": 0.13006147742271423, "median": 0.12742848694324493, "p05": 0.10484378784894943, "p95": 0.15863747894763947 }, { "label": "completed_group_4", "mean": 0.11658850312232971, "median": 0.11437725275754929, "p05": 0.09685036540031433, "p95": 0.14346592128276825 }, { "label": "completed_group_5", "mean": 0.139857679605484, "median": 0.13671082258224487, "p05": 0.10990621894598007, "p95": 0.17913517355918884 }, { "label": "completed_group_6", "mean": 0.15931037068367004, "median": 0.15141257643699646, "p05": 0.10549820214509964, "p95": 0.2391991913318634 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.043663501739502, "group": 7, "latest_source_mass": 0.08876943588256836, "layer": 25, "max_source_mass": 0.17510183155536652, "mixer_index": 49, "normalized_entropy": 0.9827943997351177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10298450291156769, "median": 0.09445159137248993, "p05": 0.06719416379928589, "p95": 0.17596006393432617 }, { "label": "completed_group_1", "mean": 0.17510183155536652, "median": 0.1808546781539917, "p05": 0.12213055044412613, "p95": 0.21415141224861145 }, { "label": "completed_group_2", "mean": 0.1414714753627777, "median": 0.14097586274147034, "p05": 0.11696422845125198, "p95": 0.16771599650382996 }, { "label": "completed_group_3", "mean": 0.12628035247325897, "median": 0.12579765915870667, "p05": 0.10614679008722305, "p95": 0.14692939817905426 }, { "label": "completed_group_4", "mean": 0.11767096817493439, "median": 0.11719106882810593, "p05": 0.10052644461393356, "p95": 0.13653282821178436 }, { "label": "completed_group_5", "mean": 0.12069195508956909, "median": 0.11935321986675262, "p05": 0.09329675883054733, "p95": 0.15252181887626648 }, { "label": "completed_group_6", "mean": 0.12702946364879608, "median": 0.12134957313537598, "p05": 0.08377101272344589, "p95": 0.18232685327529907 }, { "label": "current_group_partial", "mean": 0.08876943588256836, "median": 0.07879889756441116, "p05": 0.06865477561950684, "p95": 0.15027400851249695 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0607244968414307, "group": 7, "latest_source_mass": 0.11456632614135742, "layer": 26, "max_source_mass": 0.1498526930809021, "mixer_index": 50, "normalized_entropy": 0.9909990040771789, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1220984011888504, "median": 0.12646546959877014, "p05": 0.07202506065368652, "p95": 0.15947024524211884 }, { "label": "completed_group_1", "mean": 0.1498526930809021, "median": 0.15748712420463562, "p05": 0.08880092203617096, "p95": 0.19019576907157898 }, { "label": "completed_group_2", "mean": 0.12913095951080322, "median": 0.12894359230995178, "p05": 0.1061355471611023, "p95": 0.15167440474033356 }, { "label": "completed_group_3", "mean": 0.1179434061050415, "median": 0.118401437997818, "p05": 0.09755907952785492, "p95": 0.13774089515209198 }, { "label": "completed_group_4", "mean": 0.10775718837976456, "median": 0.10657531023025513, "p05": 0.09242597222328186, "p95": 0.12714339792728424 }, { "label": "completed_group_5", "mean": 0.12020602822303772, "median": 0.11893786489963531, "p05": 0.10092492401599884, "p95": 0.14495709538459778 }, { "label": "completed_group_6", "mean": 0.13844498991966248, "median": 0.13359181582927704, "p05": 0.10005055367946625, "p95": 0.1863655149936676 }, { "label": "current_group_partial", "mean": 0.11456632614135742, "median": 0.10762587189674377, "p05": 0.08914398401975632, "p95": 0.16366823017597198 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0388383865356445, "group": 7, "latest_source_mass": 0.08920245617628098, "layer": 26, "max_source_mass": 0.18260905146598816, "mixer_index": 51, "normalized_entropy": 0.9804740098096767, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10253133624792099, "median": 0.09456855058670044, "p05": 0.07405152916908264, "p95": 0.16842687129974365 }, { "label": "completed_group_1", "mean": 0.18260905146598816, "median": 0.19173294305801392, "p05": 0.11063728481531143, "p95": 0.22958147525787354 }, { "label": "completed_group_2", "mean": 0.14208021759986877, "median": 0.14284399151802063, "p05": 0.10792264342308044, "p95": 0.17174816131591797 }, { "label": "completed_group_3", "mean": 0.12606358528137207, "median": 0.12572120130062103, "p05": 0.10405182838439941, "p95": 0.14868314564228058 }, { "label": "completed_group_4", "mean": 0.11357533931732178, "median": 0.11289471387863159, "p05": 0.09577979147434235, "p95": 0.13378244638442993 }, { "label": "completed_group_5", "mean": 0.11596501618623734, "median": 0.1148330420255661, "p05": 0.09200432896614075, "p95": 0.1449994146823883 }, { "label": "completed_group_6", "mean": 0.12797299027442932, "median": 0.12260732054710388, "p05": 0.08455555140972137, "p95": 0.1881599724292755 }, { "label": "current_group_partial", "mean": 0.08920245617628098, "median": 0.07755134999752045, "p05": 0.066501684486866, "p95": 0.1560288667678833 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0634398460388184, "group": 7, "latest_source_mass": 0.12262405455112457, "layer": 27, "max_source_mass": 0.15090109407901764, "mixer_index": 52, "normalized_entropy": 0.9923048110176299, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12216806411743164, "median": 0.12567293643951416, "p05": 0.08283672481775284, "p95": 0.1497604101896286 }, { "label": "completed_group_1", "mean": 0.15090109407901764, "median": 0.15809711813926697, "p05": 0.09385120868682861, "p95": 0.1825941503047943 }, { "label": "completed_group_2", "mean": 0.12766355276107788, "median": 0.12779396772384644, "p05": 0.10599278658628464, "p95": 0.14822138845920563 }, { "label": "completed_group_3", "mean": 0.1134132444858551, "median": 0.11286742240190506, "p05": 0.09987462311983109, "p95": 0.12846460938453674 }, { "label": "completed_group_4", "mean": 0.10988184064626694, "median": 0.10929364711046219, "p05": 0.09590980410575867, "p95": 0.12525902688503265 }, { "label": "completed_group_5", "mean": 0.11335283517837524, "median": 0.11079756915569305, "p05": 0.09715084731578827, "p95": 0.14056096971035004 }, { "label": "completed_group_6", "mean": 0.13999530673027039, "median": 0.1355634331703186, "p05": 0.10303047299385071, "p95": 0.18968813121318817 }, { "label": "current_group_partial", "mean": 0.12262405455112457, "median": 0.11672497540712357, "p05": 0.09879282861948013, "p95": 0.16369464993476868 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0386834144592285, "group": 7, "latest_source_mass": 0.08390261977910995, "layer": 27, "max_source_mass": 0.17931805551052094, "mixer_index": 53, "normalized_entropy": 0.9803994839943029, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10029368102550507, "median": 0.09128708392381668, "p05": 0.07227826118469238, "p95": 0.17348244786262512 }, { "label": "completed_group_1", "mean": 0.17931805551052094, "median": 0.1862885057926178, "p05": 0.11921355128288269, "p95": 0.22246700525283813 }, { "label": "completed_group_2", "mean": 0.14185777306556702, "median": 0.14203041791915894, "p05": 0.10853314399719238, "p95": 0.1728457659482956 }, { "label": "completed_group_3", "mean": 0.13114282488822937, "median": 0.1317460834980011, "p05": 0.10662781447172165, "p95": 0.15405671298503876 }, { "label": "completed_group_4", "mean": 0.11872631311416626, "median": 0.11834143102169037, "p05": 0.09892343729734421, "p95": 0.1395292729139328 }, { "label": "completed_group_5", "mean": 0.11815309524536133, "median": 0.11647512018680573, "p05": 0.09220993518829346, "p95": 0.14916184544563293 }, { "label": "completed_group_6", "mean": 0.12660562992095947, "median": 0.12275825440883636, "p05": 0.08448901027441025, "p95": 0.1805192083120346 }, { "label": "current_group_partial", "mean": 0.08390261977910995, "median": 0.07283803820610046, "p05": 0.06168072670698166, "p95": 0.14687296748161316 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0626261234283447, "group": 7, "latest_source_mass": 0.1115579903125763, "layer": 28, "max_source_mass": 0.15398627519607544, "mixer_index": 54, "normalized_entropy": 0.9919134931593667, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11644256114959717, "median": 0.12025526911020279, "p05": 0.07829730212688446, "p95": 0.1473471075296402 }, { "label": "completed_group_1", "mean": 0.15398627519607544, "median": 0.1615961492061615, "p05": 0.09488844126462936, "p95": 0.18759915232658386 }, { "label": "completed_group_2", "mean": 0.13596117496490479, "median": 0.13718228042125702, "p05": 0.11204125732183456, "p95": 0.15343619883060455 }, { "label": "completed_group_3", "mean": 0.12036904692649841, "median": 0.1201494038105011, "p05": 0.10638508945703506, "p95": 0.13539429008960724 }, { "label": "completed_group_4", "mean": 0.10964121669530869, "median": 0.10774455964565277, "p05": 0.09591315686702728, "p95": 0.1285344362258911 }, { "label": "completed_group_5", "mean": 0.11886264383792877, "median": 0.11775418370962143, "p05": 0.0997074693441391, "p95": 0.14473959803581238 }, { "label": "completed_group_6", "mean": 0.13317911326885223, "median": 0.12975312769412994, "p05": 0.09947147965431213, "p95": 0.1780128926038742 }, { "label": "current_group_partial", "mean": 0.1115579903125763, "median": 0.1049065813422203, "p05": 0.0882733166217804, "p95": 0.15690873563289642 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.020051956176758, "group": 7, "latest_source_mass": 0.07486395537853241, "layer": 28, "max_source_mass": 0.1913527548313141, "mixer_index": 55, "normalized_entropy": 0.9714396465047528, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09590239822864532, "median": 0.08437805622816086, "p05": 0.062284864485263824, "p95": 0.19059056043624878 }, { "label": "completed_group_1", "mean": 0.1913527548313141, "median": 0.19941553473472595, "p05": 0.11809638142585754, "p95": 0.24551606178283691 }, { "label": "completed_group_2", "mean": 0.14906750619411469, "median": 0.14890097081661224, "p05": 0.1085248589515686, "p95": 0.18817238509655 }, { "label": "completed_group_3", "mean": 0.1341754049062729, "median": 0.13468942046165466, "p05": 0.10614670813083649, "p95": 0.16129878163337708 }, { "label": "completed_group_4", "mean": 0.11847006529569626, "median": 0.11653131991624832, "p05": 0.09301862120628357, "p95": 0.14944709837436676 }, { "label": "completed_group_5", "mean": 0.11603337526321411, "median": 0.11465917527675629, "p05": 0.08672616630792618, "p95": 0.1505887657403946 }, { "label": "completed_group_6", "mean": 0.12013453245162964, "median": 0.11430312693119049, "p05": 0.07354772090911865, "p95": 0.18300247192382812 }, { "label": "current_group_partial", "mean": 0.07486395537853241, "median": 0.06415845453739166, "p05": 0.052880410104990005, "p95": 0.13596120476722717 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0452136993408203, "group": 8, "latest_source_mass": 0.09500804543495178, "layer": 29, "max_source_mass": 0.15673108398914337, "mixer_index": 56, "normalized_entropy": 0.9835398871990577, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11420649290084839, "median": 0.11760550737380981, "p05": 0.0455118790268898, "p95": 0.1930246502161026 }, { "label": "completed_group_1", "mean": 0.15673108398914337, "median": 0.1630343794822693, "p05": 0.0941019207239151, "p95": 0.2044450342655182 }, { "label": "completed_group_2", "mean": 0.13583534955978394, "median": 0.1358843892812729, "p05": 0.10828851163387299, "p95": 0.16276802122592926 }, { "label": "completed_group_3", "mean": 0.12758757174015045, "median": 0.12776434421539307, "p05": 0.10106872022151947, "p95": 0.1536240577697754 }, { "label": "completed_group_4", "mean": 0.1112617775797844, "median": 0.10795636475086212, "p05": 0.0855436623096466, "p95": 0.1458887755870819 }, { "label": "completed_group_5", "mean": 0.13417255878448486, "median": 0.13457609713077545, "p05": 0.10089332610368729, "p95": 0.16806170344352722 }, { "label": "completed_group_6", "mean": 0.12519711256027222, "median": 0.11858057975769043, "p05": 0.08283591270446777, "p95": 0.18291912972927094 }, { "label": "completed_group_7", "mean": 0.09500804543495178, "median": 0.086635060608387, "p05": 0.07067855447530746, "p95": 0.14996609091758728 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1403071880340576, "group": 8, "latest_source_mass": 0.10776971280574799, "layer": 29, "max_source_mass": 0.1673685610294342, "mixer_index": 57, "normalized_entropy": 0.9740957797899907, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08488726615905762, "median": 0.07340645045042038, "p05": 0.057354122400283813, "p95": 0.1699385792016983 }, { "label": "completed_group_1", "mean": 0.1673685610294342, "median": 0.17044895887374878, "p05": 0.10701967775821686, "p95": 0.213656947016716 }, { "label": "completed_group_2", "mean": 0.13726133108139038, "median": 0.13688170909881592, "p05": 0.09961593151092529, "p95": 0.17454354465007782 }, { "label": "completed_group_3", "mean": 0.12220048904418945, "median": 0.12267022579908371, "p05": 0.09608018398284912, "p95": 0.14777429401874542 }, { "label": "completed_group_4", "mean": 0.10616929084062576, "median": 0.10591423511505127, "p05": 0.08670090138912201, "p95": 0.12526394426822662 }, { "label": "completed_group_5", "mean": 0.1037440150976181, "median": 0.10274936258792877, "p05": 0.08146549016237259, "p95": 0.13024258613586426 }, { "label": "completed_group_6", "mean": 0.10401669144630432, "median": 0.10373582690954208, "p05": 0.06632979214191437, "p95": 0.14398667216300964 }, { "label": "completed_group_7", "mean": 0.06658264249563217, "median": 0.05958016216754913, "p05": 0.04998533055186272, "p95": 0.10914435982704163 }, { "label": "current_group_partial", "mean": 0.10776971280574799, "median": 0.09594395756721497, "p05": 0.05985837057232857, "p95": 0.1909799873828888 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.18704891204834, "group": 8, "latest_source_mass": 0.1059001013636589, "layer": 30, "max_source_mass": 0.12684623897075653, "mixer_index": 58, "normalized_entropy": 0.9953688551489734, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10885682702064514, "median": 0.11496149003505707, "p05": 0.06692696362733841, "p95": 0.13488291203975677 }, { "label": "completed_group_1", "mean": 0.12684623897075653, "median": 0.12977978587150574, "p05": 0.0880269706249237, "p95": 0.15352922677993774 }, { "label": "completed_group_2", "mean": 0.11236055195331573, "median": 0.1118544340133667, "p05": 0.0995602011680603, "p95": 0.12739220261573792 }, { "label": "completed_group_3", "mean": 0.10673344880342484, "median": 0.10557802021503448, "p05": 0.09419171512126923, "p95": 0.12314252555370331 }, { "label": "completed_group_4", "mean": 0.10181289911270142, "median": 0.10104846954345703, "p05": 0.0905531719326973, "p95": 0.1154235228896141 }, { "label": "completed_group_5", "mean": 0.11441421508789062, "median": 0.11405807733535767, "p05": 0.1015545204281807, "p95": 0.12890149652957916 }, { "label": "completed_group_6", "mean": 0.11840946972370148, "median": 0.11774120479822159, "p05": 0.09769279509782791, "p95": 0.1386050581932068 }, { "label": "completed_group_7", "mean": 0.10466624796390533, "median": 0.10123022645711899, "p05": 0.08714763075113297, "p95": 0.1335311383008957 }, { "label": "current_group_partial", "mean": 0.1059001013636589, "median": 0.10151755809783936, "p05": 0.08478552103042603, "p95": 0.14096680283546448 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1283018589019775, "group": 8, "latest_source_mass": 0.07073106616735458, "layer": 30, "max_source_mass": 0.1812124252319336, "mixer_index": 59, "normalized_entropy": 0.9686319190376982, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08667369186878204, "median": 0.07598486542701721, "p05": 0.0599454902112484, "p95": 0.17076340317726135 }, { "label": "completed_group_1", "mean": 0.1812124252319336, "median": 0.18718186020851135, "p05": 0.11695106327533722, "p95": 0.22937621176242828 }, { "label": "completed_group_2", "mean": 0.14178472757339478, "median": 0.1410643458366394, "p05": 0.10069074481725693, "p95": 0.18480442464351654 }, { "label": "completed_group_3", "mean": 0.12440058588981628, "median": 0.12452012300491333, "p05": 0.09520045667886734, "p95": 0.15427032113075256 }, { "label": "completed_group_4", "mean": 0.11160320043563843, "median": 0.11176156997680664, "p05": 0.08930732309818268, "p95": 0.1340266317129135 }, { "label": "completed_group_5", "mean": 0.10330478101968765, "median": 0.10138857364654541, "p05": 0.07821988314390182, "p95": 0.13451263308525085 }, { "label": "completed_group_6", "mean": 0.11197599023580551, "median": 0.11038671433925629, "p05": 0.06996595859527588, "p95": 0.1621071994304657 }, { "label": "completed_group_7", "mean": 0.06831350922584534, "median": 0.06078982725739479, "p05": 0.049278710037469864, "p95": 0.11596570909023285 }, { "label": "current_group_partial", "mean": 0.07073106616735458, "median": 0.06323730200529099, "p05": 0.051058635115623474, "p95": 0.11510380357503891 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.173035144805908, "group": 8, "latest_source_mass": 0.097323939204216, "layer": 31, "max_source_mass": 0.13639645278453827, "mixer_index": 60, "normalized_entropy": 0.9889909148205337, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10752685368061066, "median": 0.11246659606695175, "p05": 0.04835075885057449, "p95": 0.15864257514476776 }, { "label": "completed_group_1", "mean": 0.13639645278453827, "median": 0.1424846053123474, "p05": 0.0804441049695015, "p95": 0.17550095915794373 }, { "label": "completed_group_2", "mean": 0.11839058995246887, "median": 0.11810846626758575, "p05": 0.0967480018734932, "p95": 0.13973747193813324 }, { "label": "completed_group_3", "mean": 0.11235916614532471, "median": 0.112240731716156, "p05": 0.09452164173126221, "p95": 0.12995943427085876 }, { "label": "completed_group_4", "mean": 0.10138258337974548, "median": 0.09962283074855804, "p05": 0.08511637151241302, "p95": 0.12233416736125946 }, { "label": "completed_group_5", "mean": 0.11953261494636536, "median": 0.12015771865844727, "p05": 0.09630562365055084, "p95": 0.14168336987495422 }, { "label": "completed_group_6", "mean": 0.11256720125675201, "median": 0.11007197946310043, "p05": 0.0816735252737999, "p95": 0.14915722608566284 }, { "label": "completed_group_7", "mean": 0.09452059864997864, "median": 0.0890384241938591, "p05": 0.07612448185682297, "p95": 0.13178984820842743 }, { "label": "current_group_partial", "mean": 0.097323939204216, "median": 0.0898042768239975, "p05": 0.07185541093349457, "p95": 0.14817608892917633 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1028335094451904, "group": 8, "latest_source_mass": 0.06014348566532135, "layer": 31, "max_source_mass": 0.19083696603775024, "mixer_index": 61, "normalized_entropy": 0.9570407736811942, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08200174570083618, "median": 0.06842087209224701, "p05": 0.05465565621852875, "p95": 0.1860819309949875 }, { "label": "completed_group_1", "mean": 0.19083696603775024, "median": 0.19397340714931488, "p05": 0.1252523958683014, "p95": 0.24377956986427307 }, { "label": "completed_group_2", "mean": 0.15512439608573914, "median": 0.15396007895469666, "p05": 0.10948964953422546, "p95": 0.2031564563512802 }, { "label": "completed_group_3", "mean": 0.1322997808456421, "median": 0.131678968667984, "p05": 0.1001092940568924, "p95": 0.16698354482650757 }, { "label": "completed_group_4", "mean": 0.11209174990653992, "median": 0.11124030500650406, "p05": 0.08621159940958023, "p95": 0.1389508843421936 }, { "label": "completed_group_5", "mean": 0.10435720533132553, "median": 0.10206517577171326, "p05": 0.07588012516498566, "p95": 0.13983221352100372 }, { "label": "completed_group_6", "mean": 0.1024213582277298, "median": 0.10071991384029388, "p05": 0.06045639514923096, "p95": 0.15181156992912292 }, { "label": "completed_group_7", "mean": 0.06072331964969635, "median": 0.05212578922510147, "p05": 0.04197835922241211, "p95": 0.10935051739215851 }, { "label": "current_group_partial", "mean": 0.06014348566532135, "median": 0.054720573127269745, "p05": 0.043509967625141144, "p95": 0.09508825838565826 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.147671937942505, "group": 8, "latest_source_mass": 0.07905417680740356, "layer": 32, "max_source_mass": 0.14984291791915894, "mixer_index": 62, "normalized_entropy": 0.9774476219204733, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09926988929510117, "median": 0.09770013391971588, "p05": 0.04949922859668732, "p95": 0.1754719465970993 }, { "label": "completed_group_1", "mean": 0.14984291791915894, "median": 0.1559048891067505, "p05": 0.08648592233657837, "p95": 0.2032528817653656 }, { "label": "completed_group_2", "mean": 0.12505242228507996, "median": 0.12578052282333374, "p05": 0.09032565355300903, "p95": 0.15844053030014038 }, { "label": "completed_group_3", "mean": 0.13004425168037415, "median": 0.12868037819862366, "p05": 0.10237257927656174, "p95": 0.1618403047323227 }, { "label": "completed_group_4", "mean": 0.1122550517320633, "median": 0.10850167274475098, "p05": 0.08462762087583542, "p95": 0.15187090635299683 }, { "label": "completed_group_5", "mean": 0.12226949632167816, "median": 0.12273561954498291, "p05": 0.08938803523778915, "p95": 0.15480002760887146 }, { "label": "completed_group_6", "mean": 0.10689134150743484, "median": 0.10258689522743225, "p05": 0.07286856323480606, "p95": 0.15216295421123505 }, { "label": "completed_group_7", "mean": 0.07532045990228653, "median": 0.06964509189128876, "p05": 0.054946303367614746, "p95": 0.11525978893041611 }, { "label": "current_group_partial", "mean": 0.07905417680740356, "median": 0.072466179728508, "p05": 0.05414725840091705, "p95": 0.11476384103298187 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.109609603881836, "group": 8, "latest_source_mass": 0.06053228676319122, "layer": 32, "max_source_mass": 0.18956288695335388, "mixer_index": 63, "normalized_entropy": 0.9601247071609755, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08386656641960144, "median": 0.07061353325843811, "p05": 0.055651482194662094, "p95": 0.18354055285453796 }, { "label": "completed_group_1", "mean": 0.18956288695335388, "median": 0.19494932889938354, "p05": 0.12378176301717758, "p95": 0.24272672832012177 }, { "label": "completed_group_2", "mean": 0.1487923562526703, "median": 0.14935529232025146, "p05": 0.10487234592437744, "p95": 0.19193747639656067 }, { "label": "completed_group_3", "mean": 0.12990039587020874, "median": 0.1300441324710846, "p05": 0.09967920184135437, "p95": 0.160600483417511 }, { "label": "completed_group_4", "mean": 0.11159682273864746, "median": 0.11003521829843521, "p05": 0.08588480204343796, "p95": 0.14072611927986145 }, { "label": "completed_group_5", "mean": 0.10738209635019302, "median": 0.10594583302736282, "p05": 0.0788765475153923, "p95": 0.14147546887397766 }, { "label": "completed_group_6", "mean": 0.10535432398319244, "median": 0.10358455777168274, "p05": 0.06328895688056946, "p95": 0.15448220074176788 }, { "label": "completed_group_7", "mean": 0.0630122721195221, "median": 0.05402897298336029, "p05": 0.044587839394807816, "p95": 0.11368448287248611 }, { "label": "current_group_partial", "mean": 0.06053228676319122, "median": 0.055282287299633026, "p05": 0.04416987672448158, "p95": 0.09472700953483582 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1622862815856934, "group": null, "latest_source_mass": 0.12579965591430664, "layer": null, "max_source_mass": 0.14956718683242798, "mixer_index": null, "normalized_entropy": 0.9840988963481907, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.12356588989496231, "median": 0.12339343130588531, "p05": 0.0884157195687294, "p95": 0.16069066524505615 }, { "label": "completed_group_1", "mean": 0.09235234558582306, "median": 0.09365912526845932, "p05": 0.06739562749862671, "p95": 0.11328895390033722 }, { "label": "completed_group_2", "mean": 0.08736803382635117, "median": 0.08676370978355408, "p05": 0.07290066033601761, "p95": 0.10374386608600616 }, { "label": "completed_group_3", "mean": 0.08735492825508118, "median": 0.08669611811637878, "p05": 0.07205022871494293, "p95": 0.10462935268878937 }, { "label": "completed_group_4", "mean": 0.08889827132225037, "median": 0.08936247229576111, "p05": 0.07074585556983948, "p95": 0.10606974363327026 }, { "label": "completed_group_5", "mean": 0.10504580289125443, "median": 0.10402922332286835, "p05": 0.0790465772151947, "p95": 0.13349474966526031 }, { "label": "completed_group_6", "mean": 0.14004790782928467, "median": 0.13968253135681152, "p05": 0.10069862753152847, "p95": 0.1819821298122406 }, { "label": "completed_group_7", "mean": 0.14956718683242798, "median": 0.14930106699466705, "p05": 0.12316055595874786, "p95": 0.17985624074935913 }, { "label": "completed_group_8", "mean": 0.12579965591430664, "median": 0.12568724155426025, "p05": 0.07226289063692093, "p95": 0.17831499874591827 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", "optimizer_state_before": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.010549778118729591, 0.013795096427202225, 0.012347182258963585, 0.012820021249353886, 0.01913420483469963, 0.018582463264465332, 0.013614931143820286, 0.013612493872642517, 0.019129201769828796, 0.02525762841105461, 0.027986859902739525, 0.014458909630775452, 0.018231205642223358, 0.016914349049329758, 0.022478405386209488, 0.03932488337159157, 0.03611915931105614, 0.0266753900796175, 0.018954744562506676, 0.014813725836575031, 0.013802599161863327, 0.016533223912119865, 0.016679473221302032, 0.01826235093176365, 0.015529165975749493, 0.011229476891458035, 0.008122270926833153, 0.008894859813153744, 0.009738311171531677, 0.007465044502168894, 0.008605006150901318, 0.009693782776594162 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.478636267573165e-05, "first_to_last_ratio": 5.2633491541899735, "imbalance_abs_log_ratio": 1.6607675454574897, "last_quartile_mean": 6.60916873584938e-06, "mean": 1.8399187894146962e-05, "normalized": [ 2.32572864627557, 2.093980749525082, 2.097253493167976, 1.9748047942119253, 1.9001199222002307, 1.7176970887283356, 1.5731226694511573, 1.442465715104865, 1.388173701809276, 1.2578149167122274, 1.163711179168628, 1.0739337627510765, 1.0625631931116428, 0.9721718406735991, 0.8978727300712377, 0.830314261296444, 0.8465768431906301, 0.7713093251343669, 0.7043450693146728, 0.6424476146199775, 0.6810764492568463, 0.62147201099682, 0.5661400138095133, 0.5212255519238714, 0.43050775556639687, 0.41135344603945795, 0.3923546966297886, 0.3784948438109706, 0.3251037643245192, 0.3141780758593552, 0.3117257919642976, 0.30996008329924213 ], "peak_layer": 1, "peak_normalized": 2.32572864627557, "population_cv": 0.5996620573317034, "rest_mean": 1.9884464530116895e-05, "spike_contrast": 0.5219498892811426, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0378694059909322e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.279151835362427e-05, 3.852754525723867e-05, 3.858776108245365e-05, 3.633480446296744e-05, 3.496066346997395e-05, 3.160423148074187e-05, 2.8944179575773887e-05, 2.6540197723079473e-05, 2.5541268769302405e-05, 2.3142772988649085e-05, 2.141134064004291e-05, 1.9759509086725302e-05, 1.955029983946588e-05, 1.7887172361952253e-05, 1.65201290656114e-05, 1.527710810478311e-05, 1.5576326404698193e-05, 1.4191465197654907e-05, 1.295937727263663e-05, 1.1820514373539481e-05, 1.2531253560155164e-05, 1.1434580301283859e-05, 1.041651648847619e-05, 9.590126865077764e-06, 7.92099308455363e-06, 7.56856934458483e-06, 7.2190077844425105e-06, 6.963997748243855e-06, 5.981645244901301e-06, 5.7806214499578346e-06, 5.73550141780288e-06, 5.7030138123082e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013640912038681563, "first_to_last_ratio": 5.121983284590993, "imbalance_abs_log_ratio": 1.6335417243184733, "last_quartile_mean": 2.663209011188883e-05, "mean": 7.298096016938871e-05, "normalized": [ 2.2789977416441083, 2.0684974265127174, 2.076163899895869, 1.9536947342428674, 1.8791834481613054, 1.70558006750568, 1.5594206835801008, 1.4313066933158733, 1.382104599488904, 1.2645011290875958, 1.1686177713576213, 1.080869560986379, 1.070406696124593, 0.9820093608320563, 0.9088928309552108, 0.8405354206057669, 0.8564067293095593, 0.7807789902359659, 0.7130468773072498, 0.650906500520924, 0.6919540061414183, 0.6322816699001862, 0.5753535339386485, 0.5291429877926532, 0.43715672246183995, 0.41718055352967753, 0.3982548633280403, 0.38497320016438824, 0.33031538135673594, 0.31952371692646125, 0.31683848721825286, 0.31510371557135003 ], "peak_layer": 1, "peak_normalized": 2.2789977416441083, "population_cv": 0.5877415594416108, "rest_mean": 7.874945926973683e-05, "spike_contrast": 0.5311917747171685, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.183106502750888e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016632344340905547, 0.00015096092829480767, 0.00015152043488342315, 0.00014258251758292317, 0.00013714461238123477, 0.00012447487097233534, 0.00011380801879568025, 0.00010445813677506521, 0.00010086732072522864, 9.228450653608888e-05, 8.528684702469036e-05, 7.888289837865159e-05, 7.811930845491588e-05, 7.166798604885116e-05, 6.633187149418518e-05, 6.134308205218986e-05, 6.25013854005374e-05, 5.698200038750656e-05, 5.2038845751667395e-05, 4.7503781388513744e-05, 5.04994677612558e-05, 4.614452336682007e-05, 4.198985334369354e-05, 3.861736331600696e-05, 3.190411734976806e-05, 3.044623736059293e-05, 2.9065022317809053e-05, 2.8095713787479326e-05, 2.4106733690132387e-05, 2.3319147658185102e-05, 2.3123177015804686e-05, 2.2996571715339087e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12096897326409817, "first_to_last_ratio": 5.121983389958884, "imbalance_abs_log_ratio": 1.6335417448901706, "last_quartile_mean": 0.02361760358326137, "mean": 0.06472024595132098, "normalized": [ 2.2789977721075467, 2.0684973368733783, 2.0761638538672402, 1.953694825115509, 1.8791834311760194, 1.7055800372311585, 1.5594207388065806, 1.4313066964307424, 1.3821046167764135, 1.264501125605702, 1.1686178592805807, 1.080869532341778, 1.0704067558461303, 0.9820093835771068, 0.9088927809148353, 0.8405354027150204, 0.8564067368028002, 0.7807789687785609, 0.7130468636740094, 0.6509064717345558, 0.6919539840797562, 0.6322817027038486, 0.5753535315685364, 0.529143012125947, 0.4371566943170802, 0.41718053416692213, 0.39825484202638917, 0.3849732137717626, 0.3303153774147322, 0.31952373248220817, 0.3168384772300972, 0.31510370845705143 ], "peak_layer": 1, "peak_normalized": 2.2789977721075467, "population_cv": 0.5877415617202747, "rest_mean": 0.06983580868001338, "spike_contrast": 0.5311917756455903, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03709620721638203, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.147497296333313, 0.13387365639209747, 0.1343698352575302, 0.12644360959529877, 0.12162121385335922, 0.11038555949926376, 0.10092609375715256, 0.09263452142477036, 0.08945015072822571, 0.08183882385492325, 0.07563323527574539, 0.06995414197444916, 0.06927698850631714, 0.06355588883161545, 0.05882376432418823, 0.05439965799450874, 0.05542685464024544, 0.050532206892967224, 0.04614856839179993, 0.04212682694196701, 0.04478343203663826, 0.04092142730951309, 0.0372370220720768, 0.03424626588821411, 0.028292888775467873, 0.027000026777386665, 0.025775151327252388, 0.024915561079978943, 0.02137809246778488, 0.0206796545535326, 0.020505864173173904, 0.02039358951151371 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.711734099302703e-06, "first_to_last_ratio": 4.6783638296609995, "imbalance_abs_log_ratio": 1.54294843975527, "last_quartile_mean": 1.648382720986774e-06, "mean": 4.007230394620365e-06, "normalized": [ 3.358984576738256, 2.372641967350387, 2.13819011970407, 1.8393394452465484, 1.59553806661366, 1.4487556048903885, 1.2955565469325476, 1.3466327346665505, 1.2739273872501113, 1.0660089247937763, 1.0478478795998651, 1.0914936255170047, 1.0411021864111147, 0.9154927446436142, 0.8065881507886284, 0.7702055746502616, 0.8215637644294851, 0.7364762575936972, 0.6730206907422001, 0.6266437238096296, 0.6693956877918972, 0.6236755529871887, 0.5897829924931797, 0.560318828703654, 0.45883664783466677, 0.44720692888245495, 0.4388849180240322, 0.4318078866173612, 0.37877623882612776, 0.3751519735069292, 0.3795957753533195, 0.38055659660739233 ], "peak_layer": 1, "peak_normalized": 3.358984576738256, "population_cv": 0.6648021914060557, "rest_mean": 4.31860559678575e-06, "spike_contrast": 0.5385544594899636, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3258043029272812e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.346022509096656e-05, 9.50772300711833e-06, 8.568220437155105e-06, 7.3706569310161285e-06, 6.39368863630807e-06, 5.805497494293377e-06, 5.19159357281751e-06, 5.3962676247465424e-06, 5.1049205467279535e-06, 4.271743364370195e-06, 4.19896787207108e-06, 4.373866431706119e-06, 4.171936325292336e-06, 3.6685903523903107e-06, 3.232184553780826e-06, 3.0863911888445728e-06, 3.292195287940558e-06, 2.9512300443457207e-06, 2.6969489681505365e-06, 2.511105776648037e-06, 2.6824227461474948e-06, 2.4992116323119262e-06, 2.363396333748824e-06, 2.245326641059364e-06, 1.838664161368797e-06, 1.7920611981026013e-06, 1.7587129832463688e-06, 1.730353687889874e-06, 1.5178436569840414e-06, 1.5033203908387804e-06, 1.5211277286653058e-06, 1.5249779607984237e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001145691167039331, "first_to_last_ratio": 4.836936726468446, "imbalance_abs_log_ratio": 1.5762816126034256, "last_quartile_mean": 2.368629634474928e-05, "mean": 6.211562191538178e-05, "normalized": [ 2.348876988858354, 2.071301897195793, 2.0294561010843566, 1.9098237582754058, 1.8088773728255583, 1.6555764779765272, 1.5234729301569887, 1.4082090466002533, 1.3570526012658433, 1.2477509254620376, 1.1584754738367655, 1.0770687308168136, 1.0629673480990054, 0.9794260409949351, 0.9103157365029115, 0.8467425542641047, 0.8606040430432489, 0.7903153532861079, 0.7260518902950809, 0.6672536282299563, 0.7127906049054458, 0.6531196817841458, 0.5952239503818174, 0.548639546167493, 0.4562347554219186, 0.43716465149452993, 0.4212539905654889, 0.40928696694430294, 0.34099585105581787, 0.33116538282585856, 0.32832835587544684, 0.32617736350768683 ], "peak_layer": 1, "peak_normalized": 2.348876988858354, "population_cv": 0.5738572576504661, "rest_mean": 6.679497912248028e-05, "spike_contrast": 0.5516446517557004, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6847092997049916e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014590195496566594, 0.00012866020551882684, 0.00012606092786882073, 0.0001186298904940486, 0.00011235954298172146, 0.00010283716255798936, 9.463146852795035e-05, 8.747178071644157e-05, 8.429416629951447e-05, 7.750482473056763e-05, 7.195942453108728e-05, 6.690279406029731e-05, 6.602687790291384e-05, 6.0837657656520605e-05, 5.6544828112237155e-05, 5.259594036033377e-05, 5.3456955356523395e-05, 4.909092967864126e-05, 4.509916470851749e-05, 4.144687409279868e-05, 4.4275431719142944e-05, 4.0568935219198465e-05, 3.697270585689694e-05, 3.4079086617566645e-05, 2.8339305572444573e-05, 2.715475420700386e-05, 2.6166453608311713e-05, 2.542311449360568e-05, 2.1181169358897023e-05, 2.0570543711073697e-05, 2.0394320017658174e-05, 2.026070978899952e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.517959642835194e-05, "first_to_last_ratio": 4.053573868997912, "imbalance_abs_log_ratio": 1.39959892878961, "last_quartile_mean": 2.3480414938603644e-05, "mean": 5.345439518578132e-05, "normalized": [ 2.3487685774234603, 2.032002289203053, 1.963516724917696, 1.8356141378280442, 1.6872738480579543, 1.5647260343512426, 1.4530976571042717, 1.3596062267041897, 1.314646683688125, 1.2118086283799, 1.12751130576606, 1.057258217084054, 1.0421641943948246, 0.9643812108591268, 0.9028111172523068, 0.8481148430618181, 0.8638417335406758, 0.7968091313416481, 0.7398800228203574, 0.6885877184509301, 0.7528185246328142, 0.6965652282580993, 0.6386689529276455, 0.5954414082316423, 0.5183367890031948, 0.5027841940839927, 0.4926901944439643, 0.48501928420581064, 0.3867176094242744, 0.3783808574805794, 0.37618160958425234, 0.373975045493992 ], "peak_layer": 1, "peak_normalized": 2.3487685774234603, "population_cv": 0.531886342087635, "rest_mean": 5.701439672352367e-05, "spike_contrast": 0.6003814623868411, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.423038688197266e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012555200373753905, 0.0001086194533854723, 0.00010495859896764159, 9.812164353206754e-05, 9.019220306072384e-05, 8.364148379769176e-05, 7.76744564063847e-05, 7.267692853929475e-05, 7.027364335954189e-05, 6.47764973109588e-05, 6.0270434914855286e-05, 5.65150985494256e-05, 5.570825669565238e-05, 5.155041435500607e-05, 4.8259222239721566e-05, 4.533546598395333e-05, 4.6176137402653694e-05, 4.259295019437559e-05, 3.9549839129904285e-05, 3.6808040022151545e-05, 4.02414589188993e-05, 3.7234472983982414e-05, 3.413966260268353e-05, 3.182896034559235e-05, 2.7707379558705725e-05, 2.687602500373032e-05, 2.633645635796711e-05, 2.5926412490662187e-05, 2.0671755919465795e-05, 2.0226119886501692e-05, 2.0108560420339927e-05, 1.99906098714564e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002512736391508952, "first_to_last_ratio": 5.815826874804025, "imbalance_abs_log_ratio": 1.7605829727324667, "last_quartile_mean": 4.320514426581212e-05, "mean": 0.00013164091467388062, "normalized": [ 2.22465467951502, 2.0950685860525193, 2.148742715865945, 2.011602148255569, 1.9594409030366386, 1.7609558829295935, 1.610634891401338, 1.4591459078647937, 1.4125272633741999, 1.2911637629028558, 1.1922164013125294, 1.0885645721953474, 1.0786559850616189, 0.9886788153600501, 0.9103044070794555, 0.8363940916690796, 0.853516847878044, 0.7731887115305843, 0.7048080081053143, 0.6396644863918306, 0.6683488269421659, 0.6076388883701213, 0.5570727160911497, 0.5013742768274315, 0.394539708177536, 0.3676631795999013, 0.3470701351915981, 0.3326099884380236, 0.30382715452412723, 0.2945209687200569, 0.29298846283259594, 0.29241662650296524 ], "peak_layer": 1, "peak_normalized": 2.22465467951502, "population_cv": 0.6145561150438986, "rest_mean": 0.0001427135030098725, "spike_contrast": 0.503448770748442, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.184893765952439e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00029285557684488595, 0.0002757967449724674, 0.0002828624565154314, 0.0002648091467563063, 0.00025794259272515774, 0.00023181384312920272, 0.00021202545030973852, 0.00019208330195397139, 0.00018594638095237315, 0.00016996997874230146, 0.0001569444575579837, 0.0001432996359653771, 0.0001419952604919672, 0.00013015058357268572, 0.00011983330477960408, 0.00011010368325514719, 0.00011235773854423314, 0.00010178326920140535, 9.278157085645944e-05, 8.420601807301864e-05, 8.798205089988187e-05, 7.99901390564628e-05, 7.333356188610196e-05, 6.600136839551851e-05, 5.193756805965677e-05, 4.839951725443825e-05, 4.568863005260937e-05, 4.378508310765028e-05, 3.9996084524318576e-05, 3.877100971294567e-05, 3.8569269236177206e-05, 3.8493992178700864e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.017611408606171608, 0.014558257535099983, 0.01080919150263071, 0.010327119380235672, 0.008616750128567219, 0.007770510856062174, 0.007606723811477423, 0.008487755432724953, 0.00865182839334011, 0.009020541794598103, 0.008682658895850182, 0.008706990629434586, 0.009003226645290852, 0.009712434373795986, 0.00981286820024252, 0.011337134055793285, 0.010840619914233685, 0.010956712998449802, 0.012231985107064247, 0.014082341454923153, 0.014436324127018452, 0.016183899715542793, 0.017385432496666908, 0.02254617027938366, 0.022119678556919098, 0.022372635081410408, 0.024847008287906647, 0.026897024363279343, 0.026162855327129364, 0.024674534797668457, 0.030845578759908676, 0.029862264171242714 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.2954689231701195e-05, "first_to_last_ratio": 5.105739778813491, "imbalance_abs_log_ratio": 1.63036535371595, "last_quartile_mean": 6.454439642311627e-06, "mean": 1.73502516958024e-05, "normalized": [ 2.118076439435723, 2.157395185923538, 2.1393573631676532, 2.0293387218518246, 1.9547446735792444, 1.7470380730869022, 1.5859544826144767, 1.463120381926734, 1.4022787424481573, 1.2651844959124576, 1.1596399794189511, 1.0756457189909705, 1.0548334272291358, 0.9617847148778298, 0.8861896603299768, 0.8166507566511427, 0.822632314310604, 0.7462779082282098, 0.6809665471807215, 0.6235405742761265, 0.6619400891948598, 0.6046031773324564, 0.5544108586463486, 0.5123283891876923, 0.43840604673891487, 0.41933885865900805, 0.40328491124319266, 0.3940604884962617, 0.3330148316096749, 0.3301665032697355, 0.32877203485044354, 0.3290236493310315 ], "peak_layer": 2, "peak_normalized": 2.157395185923538, "population_cv": 0.5996242881005125, "rest_mean": 1.878216889289669e-05, "spike_contrast": 0.5120760486362497, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.617898831493221e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.6749159335158765e-05, 3.7431349483085796e-05, 3.7118388718226925e-05, 3.5209537600167096e-05, 3.391531208762899e-05, 3.031155029020738e-05, 2.751670945144724e-05, 2.538550688768737e-05, 2.4329889129148796e-05, 2.195126944570802e-05, 2.0120045519433916e-05, 1.8662723960005678e-05, 1.830162545957137e-05, 1.6687206880305894e-05, 1.5375613656942733e-05, 1.4169096175464801e-05, 1.4272877706389409e-05, 1.2948109542776365e-05, 1.1814940990007017e-05, 1.0818585906235967e-05, 1.1484827155072708e-05, 1.049001730279997e-05, 9.619167940400075e-06, 8.88902650331147e-06, 7.606455255881883e-06, 7.2756347435642965e-06, 6.997094715188723e-06, 6.8370486587809864e-06, 5.777891146863112e-06, 5.728471933252877e-06, 5.704277555196313e-06, 5.708643129764823e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001289480715058744, "first_to_last_ratio": 4.956339714424028, "imbalance_abs_log_ratio": 1.6006675075081214, "last_quartile_mean": 2.6016794436145574e-05, "mean": 6.873570305288013e-05, "normalized": [ 2.073004605358973, 2.134365700374101, 2.1192095069414485, 2.0040524994244264, 1.9289936690931089, 1.7292296834320953, 1.569549055379467, 1.4495827774599748, 1.392480193841659, 1.2691570699511074, 1.1657818381111424, 1.084357575904709, 1.0620446910222163, 0.9734770749855297, 0.8984711717160666, 0.8270404385478125, 0.8348284439914535, 0.7567459964705125, 0.691239653970941, 0.6326487613732772, 0.6743242179293075, 0.6159548780125667, 0.5640484616188776, 0.521373530426787, 0.4457155128282148, 0.4257578292711852, 0.4106916383072167, 0.4012558017197981, 0.3390531928847147, 0.336176289510552, 0.3343946322908752, 0.33499360784988247 ], "peak_layer": 2, "peak_normalized": 2.134365700374101, "population_cv": 0.5868619217766903, "rest_mean": 7.428186829740837e-05, "spike_contrast": 0.5221517931823592, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.878641073242761e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001424894289812073, 0.0001467071269871667, 0.00014566535537596792, 0.0001377499575028196, 0.00013259073602966964, 0.0001188598180306144, 0.00010788405779749155, 9.963809134205803e-05, 9.571310511091724e-05, 8.723640348762274e-05, 8.013083424884826e-05, 7.4534080340527e-05, 7.300038851099089e-05, 6.691263115499169e-05, 6.175704766064882e-05, 5.68472059967462e-05, 5.7382520026294515e-05, 5.2015468099853024e-05, 4.751284359372221e-05, 4.3485557398526e-05, 4.6350149204954505e-05, 4.233809158904478e-05, 3.877026756526902e-05, 3.5836976167047396e-05, 3.0636569135822356e-05, 2.926476372522302e-05, 2.8229178496985696e-05, 2.7580599635257386e-05, 2.330505958525464e-05, 2.3107313609216362e-05, 2.2984850147622637e-05, 2.3026021153782494e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11435244046151638, "first_to_last_ratio": 4.956339696350975, "imbalance_abs_log_ratio": 1.6006675038616696, "last_quartile_mean": 0.023071953793987632, "mean": 0.0609555093687959, "normalized": [ 2.073004695896629, 2.1343657739438058, 2.1192095214175937, 2.004052535226019, 1.9289935285682593, 1.7292296759967316, 1.5695490131514764, 1.4495827726824428, 1.3924801813690295, 1.269157021277261, 1.165781889125704, 1.0843576426531487, 1.0620447122638885, 0.9734770498631476, 0.8984712131746528, 0.8270404152126933, 0.8348284100671275, 0.7567460489501358, 0.691239607862424, 0.6326487463288526, 0.6743241627324319, 0.6159549038523712, 0.5640484241789988, 0.5213735345830548, 0.44571554071981845, 0.4257578260668156, 0.4106916263563641, 0.40125582081652367, 0.3390531599397356, 0.3361762979130643, 0.33439464558088794, 0.33499360222891106 ], "peak_layer": 2, "peak_normalized": 2.1343657739438058, "population_cv": 0.5868619269179132, "rest_mean": 0.06587390429167836, "spike_contrast": 0.522151786129607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03439617678523064, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.12636105716228485, 0.13010135293006897, 0.1291774958372116, 0.12215804308652878, 0.1175827831029892, 0.10540607571601868, 0.09567265957593918, 0.08836005628108978, 0.08487933874130249, 0.07736211270093918, 0.07106082886457443, 0.06609757244586945, 0.0647374764084816, 0.05933878943324089, 0.05476677045226097, 0.05041266977787018, 0.05088739097118378, 0.046127840876579285, 0.04213486239314079, 0.038563426584005356, 0.041103772819042206, 0.03754584491252899, 0.03438185900449753, 0.03178058937191963, 0.027168817818164825, 0.025952285155653954, 0.025033917278051376, 0.0244587529450655, 0.02066715806722641, 0.020491797477006912, 0.020383195951581, 0.020419705659151077 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.461751238224679e-06, "first_to_last_ratio": 4.607278415178689, "imbalance_abs_log_ratio": 1.527637317203245, "last_quartile_mean": 1.6195572669630565e-06, "mean": 3.872976176211296e-06, "normalized": [ 3.025668897657687, 2.420035163018765, 2.225735559561749, 1.8932958237601154, 1.6740889922677953, 1.49074225689019, 1.302073782365782, 1.3813151404849735, 1.2950916709377387, 1.0715944778616184, 1.0585199120860045, 1.1104678074006644, 1.039542517960571, 0.9298369507481301, 0.7931357450923319, 0.7672068262710003, 0.8115868243354613, 0.7239540167999846, 0.6595190440563047, 0.6140523657384757, 0.6490685381015874, 0.6059744156497426, 0.5735242411534173, 0.5386197309103437, 0.45666220052738465, 0.4487521252190139, 0.4444189656933061, 0.44116761367860935, 0.38204877349263455, 0.3857109035588071, 0.3910260087310755, 0.39556270798873516 ], "peak_layer": 1, "peak_normalized": 3.025668897657687, "population_cv": 0.6453832806768665, "rest_mean": 4.185131009156369e-06, "spike_contrast": 0.5226455452702347, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.187340078307898e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1718343557731714e-05, 9.372738531965297e-06, 8.620220796728972e-06, 7.332689619943267e-06, 6.483706783910748e-06, 5.773609245807165e-06, 5.042900738772005e-06, 5.349800630938262e-06, 5.015859187551541e-06, 4.150259883317631e-06, 4.09962240155437e-06, 4.300815362512367e-06, 4.026123406219995e-06, 3.601236358008464e-06, 3.0717958452441962e-06, 2.971373760374263e-06, 3.1432564355782233e-06, 2.8038566597388126e-06, 2.554301545387716e-06, 2.3782101834513014e-06, 2.5138269847957417e-06, 2.3469244752050145e-06, 2.221245722466847e-06, 2.0860613858530996e-06, 1.7686418232187862e-06, 1.7380062899974291e-06, 1.7212240663866396e-06, 1.7086316574932425e-06, 1.4796657978877192e-06, 1.4938491403881926e-06, 1.5144344160944456e-06, 1.5320049442379968e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00010837215813808143, "first_to_last_ratio": 4.663749544487092, "imbalance_abs_log_ratio": 1.5398197478822726, "last_quartile_mean": 2.3237130790221272e-05, "mean": 5.872993017419503e-05, "normalized": [ 2.116908005707516, 2.1257566114370654, 2.0713368933145166, 1.958807069139986, 1.853064595344867, 1.6774795353818415, 1.534246942340664, 1.4245031207093415, 1.3659761301608329, 1.2527187093598229, 1.1583387848911404, 1.08159561433793, 1.0590461894491214, 0.974994469979534, 0.9039318927032919, 0.837094157953219, 0.8447810620709018, 0.7717232000223526, 0.7087798354672495, 0.6523243210626272, 0.6962576906319006, 0.6381512392879863, 0.585489683522781, 0.5414078956394395, 0.4665886811070132, 0.44848929689996414, 0.43561388426614667, 0.42649458236277493, 0.3502420718536938, 0.3476010806933041, 0.3452465505067908, 0.3450102023943839 ], "peak_layer": 2, "peak_normalized": 2.1257566114370654, "population_cv": 0.5682342302969126, "rest_mean": 6.323713649989158e-05, "spike_contrast": 0.5438420826580702, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4391016015433706e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012432585936039686, 0.0001248455373570323, 0.00012164947111159563, 0.000115040602395311, 0.00010883035429287702, 9.851825598161668e-05, 9.010621579363942e-05, 8.366096881218255e-05, 8.022368274396285e-05, 7.357208232861012e-05, 6.802915595471859e-05, 6.352203490678221e-05, 6.219770875759423e-05, 5.7261357142124325e-05, 5.308785694069229e-05, 4.916248144581914e-05, 4.961393278790638e-05, 4.532324965111911e-05, 4.162659024586901e-05, 3.831096182693727e-05, 4.089116555405781e-05, 3.747857772395946e-05, 3.4385768231004477e-05, 3.179684790666215e-05, 2.7402720661484636e-05, 2.633974509080872e-05, 2.558357300586067e-05, 2.5047997041838244e-05, 2.0569692424032837e-05, 2.0414587197592482e-05, 2.0276305804145522e-05, 2.0262425096007064e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.030398450704524e-05, "first_to_last_ratio": 3.89627084299362, "imbalance_abs_log_ratio": 1.360019901586191, "last_quartile_mean": 2.3177029561338713e-05, "mean": 5.084823260403937e-05, "normalized": [ 2.109723573221063, 2.0821310668378783, 1.9942546197542355, 1.874953814933449, 1.728017146958865, 1.5814957309589324, 1.4629098120324804, 1.3741245898657877, 1.3101737824863438, 1.2101890628861518, 1.1282931314057325, 1.0632895987411368, 1.0440174426577644, 0.9649796238381606, 0.9017142254727002, 0.8410904420982939, 0.8516872374927745, 0.7851358108806284, 0.7291665843144397, 0.6770579705243002, 0.7378064582735888, 0.6786248956615917, 0.6313013382664657, 0.5913982993163636, 0.5312183147012203, 0.5201489591253866, 0.5114845807208003, 0.5059300847390553, 0.39756423665978435, 0.3954241221367383, 0.3931178424325881, 0.39157560060529917 ], "peak_layer": 1, "peak_normalized": 2.109723573221063, "population_cv": 0.5213662772063792, "rest_mean": 5.429395497668776e-05, "spike_contrast": 0.5938291252788923, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.224133179173805e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010727571498136967, 0.00010587268479866907, 0.00010140432277694345, 9.5338087703567e-05, 8.786661783233285e-05, 8.041626279009506e-05, 7.438637840095907e-05, 6.987180677242577e-05, 6.662002124357969e-05, 6.153597496449947e-05, 5.7371711591258645e-05, 5.406639684224501e-05, 5.308644176693633e-05, 4.90675083710812e-05, 4.585057467920706e-05, 4.276796244084835e-05, 4.330679075792432e-05, 3.9922768337419257e-05, 3.7076832086313516e-05, 3.4427201171638444e-05, 3.751615440705791e-05, 3.450687654549256e-05, 3.210055729141459e-05, 3.007155828527175e-05, 2.7011512429453433e-05, 2.6448655262356624e-05, 2.60080869338708e-05, 2.572565063019283e-05, 2.021543878072407e-05, 2.010661773965694e-05, 1.998934749281034e-05, 1.991092722164467e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00023694485207670368, "first_to_last_ratio": 5.643445556811249, "imbalance_abs_log_ratio": 1.7304947933060113, "last_quartile_mean": 4.198584883852163e-05, "mean": 0.00012326974319876172, "normalized": [ 2.0534243627746784, 2.17548929765909, 2.1923221930833754, 2.045443753327701, 2.026125682616779, 1.7904266625829455, 1.6133287652163053, 1.4807635751829418, 1.4348095534638061, 1.3014197885977827, 1.1747669830911103, 1.0855875327354585, 1.0626766295217445, 0.9759316522493466, 0.8962992674239908, 0.8231145619826911, 0.8258276905981559, 0.7455975627999764, 0.6776587351559191, 0.6226196341065754, 0.6492096639107944, 0.5955964221479619, 0.5398643713108281, 0.4868843463557821, 0.3972339380481051, 0.3723774041823533, 0.3563689360290182, 0.3460491488507791, 0.3146328182140966, 0.31212742697325935, 0.31138985452451795, 0.31463178528213015 ], "peak_layer": 3, "peak_normalized": 2.1923221930833754, "population_cv": 0.61857583014248, "rest_mean": 0.00013391299553741528, "spike_contrast": 0.4913352905442922, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.579618057003244e-05, "top_five_layers": [ 3, 2, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002531250938773155, 0.0002681720070540905, 0.0002702469937503338, 0.000252141326200217, 0.00024975999258458614, 0.0002207054349128157, 0.00019887462258338928, 0.00018253334565088153, 0.00017686860519461334, 0.00016042568313423544, 0.00014481322432402521, 0.0001338200963800773, 0.0001309958752244711, 0.00012030284415232018, 0.00011048658052459359, 0.00010146512067876756, 0.00010179956734646112, 9.19096200959757e-05, 8.353481825906783e-05, 7.675016240682453e-05, 8.002790855243802e-05, 7.341901800828055e-05, 6.654894241364673e-05, 6.001810834277421e-05, 4.896692553302273e-05, 4.590286698658019e-05, 4.3929507228313014e-05, 4.2657389712985605e-05, 3.878470670315437e-05, 3.8475867768283933e-05, 3.83849474019371e-05, 3.878457937389612e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.010549778118729591, 0.02910016104578972, 0.04255657270550728, 0.05177471786737442, 0.01913420483469963, 0.02246284671127796, 0.029843268916010857, 0.036163609474897385, 0.019129201769828796, 0.0314205139875412, 0.03302976116538048, 0.03871368616819382, 0.018231205642223358, 0.02580643631517887, 0.0349617674946785, 0.04659247398376465, 0.03611915931105614, 0.03664228320121765, 0.04177439957857132, 0.04776441678404808, 0.013802599161863327, 0.02693362347781658, 0.03932841867208481, 0.0514267161488533, 0.015529165975749493, 0.03558400645852089, 0.05707564577460289, 0.08332383632659912, 0.009738311171531677, 0.03187238425016403, 0.05786185711622238, 0.09122034162282944 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.478662188172166e-05, "first_to_last_ratio": 5.263389233448399, "imbalance_abs_log_ratio": 1.6607751602108825, "last_quartile_mean": 6.609167655824422e-06, "mean": 1.839926902391653e-05, "normalized": [ 2.3257405363021224, 2.093986543353821, 2.0972432569142194, 1.9748429471145472, 1.9001738268978305, 1.7176742899564605, 1.5731126681977952, 1.4424450196971237, 1.3881676796557327, 1.2578123363623133, 1.1637254248597446, 1.0739089583500412, 1.0625597930502229, 0.9721654778743629, 0.897852656475923, 0.8303134671066331, 0.8465754829823255, 0.7713090877071682, 0.7043469066766334, 0.642452591912055, 0.6810739404273755, 0.621476487606478, 0.5661443389519891, 0.5212309648706807, 0.4305042754919776, 0.4113531398630616, 0.3923510140548581, 0.378496313745133, 0.3251027262595914, 0.314172044005768, 0.31172491175027217, 0.3099608915257388 ], "peak_layer": 1, "peak_normalized": 2.3257405363021224, "population_cv": 0.5996654929960216, "rest_mean": 1.9884546603592485e-05, "spike_contrast": 0.5219515586938898, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.037877009366639e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.2791925807250664e-05, 3.852782174362801e-05, 3.858774289255962e-05, 3.633566666394472e-05, 3.4961809433298185e-05, 3.160395135637373e-05, 2.8944123187102377e-05, 2.653993396961596e-05, 2.554127058829181e-05, 2.314282755833119e-05, 2.1411697161966003e-05, 1.9759139831876382e-05, 1.9550323486328125e-05, 1.7887134163174778e-05, 1.6519832570338622e-05, 1.5277160855475813e-05, 1.5576370060443878e-05, 1.419152340531582e-05, 1.2959468222106807e-05, 1.1820658073702361e-05, 1.2531262655102182e-05, 1.1434713087510318e-05, 1.0416641998745035e-05, 9.59026874625124e-06, 7.920963980723172e-06, 7.568597084173234e-06, 7.2189718594017904e-06, 6.964055501157418e-06, 5.9816525208589155e-06, 5.780535957455868e-06, 5.735510512749897e-06, 5.703053830075078e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013640988436236512, "first_to_last_ratio": 5.122011664762131, "imbalance_abs_log_ratio": 1.6335472651589908, "last_quartile_mean": 2.6632091703504557e-05, "mean": 7.298115508547198e-05, "normalized": [ 2.2789966397783896, 2.0685142340171683, 2.076140409583827, 1.9537250082825501, 1.8792316671609142, 1.7055529809049683, 1.5594088420998298, 1.4313187223467239, 1.382107687555985, 1.2644854892365605, 1.1686221274742359, 1.0808639824268664, 1.0704039370079022, 0.9820006566284564, 0.9088834247617501, 0.8405321787627472, 0.856406336273156, 0.7807790484250425, 0.7130462191259114, 0.650909996157966, 0.6919497653761542, 0.6322822243882543, 0.5753541906206389, 0.5291452134901508, 0.4371545081034775, 0.4171814083388854, 0.3982526780954131, 0.3849724710776064, 0.3303152718064163, 0.31952201613190695, 0.31683873767491905, 0.31510192688522626 ], "peak_layer": 1, "peak_normalized": 2.2789966397783896, "population_cv": 0.5877439115216546, "rest_mean": 7.874967775057519e-05, "spike_contrast": 0.5311911602534674, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1831132693914694e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016632380720693618, 0.00015096255810931325, 0.0001515191252110526, 0.00014258510782383382, 0.00013714849774260074, 0.00012447322660591453, 0.00011380745854694396, 0.00010445929365232587, 0.0001008678154903464, 9.228361159330234e-05, 8.528739272151142e-05, 7.8882701927796e-05, 7.81193157308735e-05, 7.16675422154367e-05, 6.633136217715219e-05, 6.134300929261371e-05, 6.250152364373207e-05, 5.6982156820595264e-05, 5.203893670113757e-05, 4.750416337628849e-05, 5.049929313827306e-05, 4.6144687075866386e-05, 4.199001341476105e-05, 3.8617628888459876e-05, 3.190404095221311e-05, 3.044638106075581e-05, 2.9064940463285893e-05, 2.809573561535217e-05, 2.4106790078803897e-05, 2.331908581254538e-05, 2.312325705133844e-05, 2.2996502593741752e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12096964661031961, "first_to_last_ratio": 5.122011546850009, "imbalance_abs_log_ratio": 1.6335472421383241, "last_quartile_mean": 0.023617605213075876, "mean": 0.06472041795495898, "normalized": [ 2.278996550367558, 2.068514172728323, 2.0761403775307596, 1.9537250896945753, 1.879231622178916, 1.7055529409943957, 1.5594087663016845, 1.431318663899547, 1.382107735687561, 1.2644854472317082, 1.1686221211602732, 1.0808640120306183, 1.0704040262086971, 0.9820006724164564, 0.9088834582355364, 0.8405322479178425, 0.8564063602524741, 0.7807790810183837, 0.713046234965646, 0.6509099797949671, 0.6919497851652032, 0.6322822095960979, 0.575354189755429, 0.529145232117261, 0.4371544964362311, 0.41718141126177893, 0.3982526899729607, 0.38497247844984545, 0.33031527661149607, 0.3195220199067584, 0.3168387288221828, 0.3151019212888319 ], "peak_layer": 1, "peak_normalized": 2.278996550367558, "population_cv": 0.5877438959595169, "rest_mean": 0.06983600149827975, "spike_contrast": 0.5311911625115104, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.037096266821026805, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.14749760925769806, 0.13387510180473328, 0.1343686729669571, 0.12644590437412262, 0.121624656021595, 0.11038409918546677, 0.10092558711767197, 0.09263554215431213, 0.08945059031248093, 0.08183802664279938, 0.0756337121129036, 0.06995397061109543, 0.06927699595689774, 0.06355549395084381, 0.05882331728935242, 0.05439959838986397, 0.05542697757482529, 0.050532348453998566, 0.04614865034818649, 0.04212716594338417, 0.04478327929973602, 0.040921568870544434, 0.03723716363310814, 0.034246500581502914, 0.0282928217202425, 0.027000155299901962, 0.025775080546736717, 0.024915579706430435, 0.02137814275920391, 0.020679598674178123, 0.020505934953689575, 0.020393528044223785 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.71177167280257e-06, "first_to_last_ratio": 4.678295917137282, "imbalance_abs_log_ratio": 1.542933923351862, "last_quartile_mean": 1.6484146811990286e-06, "mean": 4.007248069370917e-06, "normalized": [ 3.3590280906213086, 2.3726088061078587, 2.138143013037793, 1.8393897753025719, 1.595536022354422, 1.4487646483149947, 1.2955498112963995, 1.3466260007113127, 1.2739769202114613, 1.0659628023122865, 1.0478341793654378, 1.0914579443825887, 1.0411457104586188, 0.9155146938805809, 0.8066044523853629, 0.7701707432135287, 0.8215762550934921, 0.7364785130598478, 0.6730090976811546, 0.6266208737012697, 0.6693583504854638, 0.6236588439557476, 0.5897891291930596, 0.5603290672032583, 0.4588325813842428, 0.4472228580494504, 0.4388965999812878, 0.43180771263272905, 0.37877595830431615, 0.3751711426274845, 0.3796103005144005, 0.3805491021762693 ], "peak_layer": 1, "peak_normalized": 3.3590280906213086, "population_cv": 0.6648044266247514, "rest_mean": 4.318630839474348e-06, "spike_contrast": 0.5385459413556807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.325781110812386e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.3460458831104916e-05, 9.507632057648152e-06, 8.568069461034611e-06, 7.370891125901835e-06, 6.393708645191509e-06, 5.805559339933097e-06, 5.191589480091352e-06, 5.396264441515086e-06, 5.105141553940484e-06, 4.271577381587122e-06, 4.198931492283009e-06, 4.373742740426678e-06, 4.172129138169112e-06, 3.6686944895336637e-06, 3.232264134567231e-06, 3.086265223828377e-06, 3.292259862064384e-06, 2.9512520995922387e-06, 2.696914407351869e-06, 2.5110252863669302e-06, 2.682284957700176e-06, 2.499155698387767e-06, 2.3634313492948422e-06, 2.2453775727626635e-06, 1.8386559759164811e-06, 1.7921329344972037e-06, 1.7587675529284752e-06, 1.730360622786975e-06, 1.5178492276390898e-06, 1.5034038369776681e-06, 1.5211926438496448e-06, 1.5249546549966908e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00011456928586994763, "first_to_last_ratio": 4.836938296613537, "imbalance_abs_log_ratio": 1.5762819372189676, "last_quartile_mean": 2.3686323629590333e-05, "mean": 6.211567796299278e-05, "normalized": [ 2.3488758065274955, 2.0713117417995783, 2.0294413849686572, 1.9098281260734524, 1.808891788235116, 1.6555687759458058, 1.5234656987314679, 1.4082197237907075, 1.3570554765296423, 1.2477441770945124, 1.1584743113969003, 1.07706541625456, 1.062967326057557, 0.9794184805192363, 0.9103158522005721, 0.8467437815449563, 0.8606016851807378, 0.7903183885158978, 0.726053695018778, 0.6672539632450412, 0.7127881461452974, 0.6531207323660743, 0.5952249360680494, 0.548642506624817, 0.4562346365956528, 0.4371660140704956, 0.4212530247856756, 0.40928721260242446, 0.3409968025795472, 0.33116514257975266, 0.32832896742236684, 0.3261762785291743 ], "peak_layer": 1, "peak_normalized": 2.3488758065274955, "population_cv": 0.5738573169932581, "rest_mean": 6.679503382690234e-05, "spike_contrast": 0.5516451476522887, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6847156297881153e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014590201317332685, 0.00012866093311458826, 0.00012606012751348317, 0.00011863026884384453, 0.0001123605397879146, 0.00010283677693223581, 9.463110473006964e-05, 8.747252286411822e-05, 8.429442095803097e-05, 7.750447548460215e-05, 7.195941725512967e-05, 6.690264854114503e-05, 6.602693611057475e-05, 6.08372429269366e-05, 5.654488631989807e-05, 5.259606405161321e-05, 5.34568571310956e-05, 4.9091162509284914e-05, 4.509931750362739e-05, 4.1446932300459594e-05, 4.4275318941799924e-05, 4.0569037082605064e-05, 3.697280044434592e-05, 3.4079301258316264e-05, 2.833932376233861e-05, 2.7154863346368074e-05, 2.6166417228523642e-05, 2.5423152692383155e-05, 2.1181247575441375e-05, 2.0570547349052504e-05, 2.0394376406329684e-05, 2.0260660676285625e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.518698971078265e-05, "first_to_last_ratio": 4.0537764308209505, "imbalance_abs_log_ratio": 1.399648898709643, "last_quartile_mean": 2.3481065454689087e-05, "mean": 5.345527347344614e-05, "normalized": [ 2.3490757134047047, 2.03180856162592, 1.9640798219543283, 1.8355777168881, 1.687288592851433, 1.564711486655573, 1.4531995506450504, 1.3597364706007256, 1.3148292531115187, 1.211773745536977, 1.127520479381137, 1.0571877619151735, 1.041978223119568, 0.9643430432278, 0.9029106186893467, 0.848047824187126, 0.863550005931762, 0.7965670293846996, 0.7398915500109849, 0.6885201220036962, 0.7527468103098911, 0.696422774701012, 0.6386525384719246, 0.5954551044265526, 0.5183504589787767, 0.502904662044204, 0.4927615213178117, 0.48500410119101467, 0.3866832502775355, 0.3782793274311306, 0.37617794688759315, 0.3739639328369289 ], "peak_layer": 1, "peak_normalized": 2.3490757134047047, "population_cv": 0.5319468462674285, "rest_mean": 5.701573583657456e-05, "spike_contrast": 0.6003391206010799, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4228776712552644e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012557048466987908, 0.00010861088230740279, 0.00010499042400624603, 9.812130883801728e-05, 9.019447315949947e-05, 8.364208042621613e-05, 7.768117939122021e-05, 7.268508488778025e-05, 7.028455729596317e-05, 6.477569695562124e-05, 6.027191557222977e-05, 5.651226092595607e-05, 5.5699230870231986e-05, 5.154922109795734e-05, 4.826533404411748e-05, 4.5332628360483795e-05, 4.6161301725078374e-05, 4.2580708395689726e-05, 3.955110514652915e-05, 3.680503141367808e-05, 4.0238286601379514e-05, 3.722746987477876e-05, 3.413934609852731e-05, 3.1830215448280796e-05, 2.7708565539796837e-05, 2.6882906240643933e-05, 2.6340701879234985e-05, 2.5926026864908636e-05, 2.0670258891186677e-05, 2.022102489718236e-05, 2.010869502555579e-05, 1.9990344299003482e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002512551309337141, "first_to_last_ratio": 5.815353323084221, "imbalance_abs_log_ratio": 1.7605015447583472, "last_quartile_mean": 4.320547986935708e-05, "mean": 0.0001316397593882357, "normalized": [ 2.224752247026743, 2.0950471769673564, 2.148762457826491, 2.0117752264198114, 1.9594249362752976, 1.7610216345575365, 1.6099537081187654, 1.4585175613957049, 1.4124641586885858, 1.2915649811078755, 1.192264449118915, 1.0889144888939652, 1.0788791319896314, 0.988941963176937, 0.9104012729618977, 0.8363983367426334, 0.8532666616213362, 0.773042726053889, 0.7048431559693451, 0.6396986756467042, 0.668255590231208, 0.6075232312555597, 0.5571337058341255, 0.5014728599784464, 0.3945136003262428, 0.3676589998419824, 0.34703606615034643, 0.33261343254293874, 0.3038165004585062, 0.2945782724854268, 0.2930231193739135, 0.29243967096188167 ], "peak_layer": 1, "peak_normalized": 2.224752247026743, "population_cv": 0.6145445602222892, "rest_mean": 0.0001427126183073226, "spike_contrast": 0.503433558134641, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.184632122516632e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00029286585049703717, 0.0002757915062829852, 0.00028286257293075323, 0.0002648296067491174, 0.00025793822715058923, 0.00023182046425063163, 0.0002119339187629521, 0.00019199890084564686, 0.00018593644199427217, 0.00017002130334731191, 0.00015694940520916134, 0.00014334444131236523, 0.00014202338934410363, 0.00013018408208154142, 0.00011984500451944768, 0.0001101032758015208, 0.00011232381802983582, 0.00010176315845455974, 9.278538345824927e-05, 8.420977974310517e-05, 8.796900510787964e-05, 7.997421198524535e-05, 7.334094698308036e-05, 6.601376662729308e-05, 5.193367542233318e-05, 4.839854227611795e-05, 4.5683744247071445e-05, 4.3785152229247615e-05, 3.999433101853356e-05, 3.877821291098371e-05, 3.857349292957224e-05, 3.849668792099692e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.021530626341700554, 0.03857670724391937, 0.04954109713435173, 0.057694368064403534, 0.02140580676496029, 0.025963041931390762, 0.03339661285281181, 0.040483180433511734, 0.02151828072965145, 0.035047177225351334, 0.03725159168243408, 0.043239567428827286, 0.02124657854437828, 0.03028978779911995, 0.03979094699025154, 0.05222286656498909, 0.03916332125663757, 0.04088718071579933, 0.04756740853190422, 0.05542457476258278, 0.020541559904813766, 0.03680417314171791, 0.05126549303531647, 0.06891486048698425, 0.02905990369617939, 0.05358703061938286, 0.07880406081676483, 0.10752817988395691, 0.029148343950510025, 0.054091889411211014, 0.08628532290458679, 0.11884573101997375 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.2954461630652077e-05, "first_to_last_ratio": 5.105695837816317, "imbalance_abs_log_ratio": 1.6303567474829335, "last_quartile_mean": 6.454450613091467e-06, "mean": 1.7350209816413553e-05, "normalized": [ 2.118074842246104, 2.1574284903940977, 2.139357285099935, 2.0292905713578766, 1.9547395369559644, 1.7470193301554893, 1.5858965602051887, 1.4631504379843852, 1.4022879982430452, 1.2651865013786623, 1.1596469721061315, 1.075647057275218, 1.0548378604659292, 0.9617861976864589, 0.8861851944905509, 0.81666640942737, 0.8226291628131417, 0.7462778224571112, 0.6809633158339518, 0.623540506764385, 0.6619488684789228, 0.6046004431197245, 0.5544183299835491, 0.5123407388295048, 0.4384090182726579, 0.4193402639944847, 0.4032847838635732, 0.3940637985584808, 0.33301498018274, 0.33016832240151506, 0.32877371956677426, 0.3290246794070765 ], "peak_layer": 2, "peak_normalized": 2.1574284903940977, "population_cv": 0.5996208655691896, "rest_mean": 1.8782105026157632e-05, "spike_contrast": 0.5120818816847563, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.61797568379552e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.674904291983694e-05, 3.7431836972245947e-05, 3.711829776875675e-05, 3.52086171915289e-05, 3.391514110262506e-05, 3.0311151931528002e-05, 2.7515638066688552e-05, 2.5385967092006467e-05, 2.4329990992555395e-05, 2.1951251255813986e-05, 2.0120118279010057e-05, 1.8662702132132836e-05, 1.8301658201380633e-05, 1.6687192328390665e-05, 1.537549906061031e-05, 1.4169333553581964e-05, 1.4272788575908635e-05, 1.2948076800967101e-05, 1.1814856406999752e-05, 1.0818558621394914e-05, 1.148495175584685e-05, 1.0489944543223828e-05, 9.619274351280183e-06, 8.889219316188246e-06, 7.606488452438498e-06, 7.27564156477456e-06, 6.997075615799986e-06, 6.837089586042566e-06, 5.77787977817934e-06, 5.728489668399561e-06, 5.704293016606243e-06, 5.708647222490981e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00012894753399450565, "first_to_last_ratio": 4.956313899688395, "imbalance_abs_log_ratio": 1.600662299067145, "last_quartile_mean": 2.601682149361295e-05, "mean": 6.873563796716553e-05, "normalized": [ 2.0729820101052394, 2.1343785185339303, 2.1192055857825096, 2.004033226215171, 1.9290001532413559, 1.7292283569198776, 1.5695265126752398, 1.449584785193964, 1.392505223726082, 1.2691464160417294, 1.1657826244262586, 1.0843620958713254, 1.0620513069452608, 0.9734787377496757, 0.8984695878304712, 0.827044661933948, 0.8348303459591176, 0.7567409969045322, 0.6912396733798123, 0.6326506836056272, 0.674324856446311, 0.615953502956551, 0.5640541296457265, 0.5213814868377082, 0.4457146646251511, 0.42575966145291916, 0.4106923182898804, 0.40125618166856064, 0.3390529581992065, 0.3361781427218294, 0.33439592808042723, 0.33499466603460004 ], "peak_layer": 2, "peak_normalized": 2.1343785185339303, "population_cv": 0.5868596491376813, "rest_mean": 7.428176730981149e-05, "spike_contrast": 0.5221542367874475, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.8786539516877386e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014248774095904082, 0.00014670786913484335, 0.00014566494792234153, 0.00013774850231129676, 0.00013259105617180467, 0.00011885961430380121, 0.00010788240615511313, 9.963813499780372e-05, 9.571473492542282e-05, 8.723558858036995e-05, 8.013081242097542e-05, 7.453432044712827e-05, 7.300077413674444e-05, 6.691268208669499e-05, 6.17568803136237e-05, 5.684744246536866e-05, 5.7382596423849463e-05, 5.201507519814186e-05, 4.7512799937976524e-05, 4.3485648347996175e-05, 4.6350149204954505e-05, 4.233795698382892e-05, 3.877062044921331e-05, 3.5837489122059196e-05, 3.0636481824330986e-05, 2.926486195065081e-05, 2.8229198505869135e-05, 2.7580599635257386e-05, 2.3305021386477165e-05, 2.3107419110601768e-05, 2.298491745023057e-05, 2.3026072085485794e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11435196176171303, "first_to_last_ratio": 4.956313746493952, "imbalance_abs_log_ratio": 1.6006622681581977, "last_quartile_mean": 0.023071978008374572, "mean": 0.06095545022981241, "normalized": [ 2.072982016679514, 2.1343786009397814, 2.119205465978219, 2.004033333779568, 1.9290000448154139, 1.7292284201771326, 1.5695265788612092, 1.4495847902174488, 1.392505122730498, 1.2691463963100211, 1.1657826534776023, 1.0843621171351583, 1.0620512430067517, 0.97347872771027, 0.8984695791578813, 0.8270447011604714, 0.8348303200871255, 0.7567409772234059, 0.691239667353491, 0.6326506435390216, 0.6743248169617974, 0.6159534846579819, 0.5640541661909846, 0.5213814964456189, 0.4457146897387812, 0.4257596753388107, 0.4106923303842507, 0.4012562101149584, 0.33905296941192775, 0.3361781519455022, 0.33439594785006815, 0.3349946606193331 ], "peak_layer": 2, "peak_normalized": 2.1343786009397814, "population_cv": 0.586859647417989, "rest_mean": 0.0658738130910529, "spike_contrast": 0.5221542395241354, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03439629077911377, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.12635955214500427, 0.1301020085811615, 0.12917712330818176, 0.12215675413608551, 0.11758306622505188, 0.10540589690208435, 0.09567119926214218, 0.08836009353399277, 0.0848807767033577, 0.07736138999462128, 0.07106080651283264, 0.06609778106212616, 0.06473781168460846, 0.05933883413672447, 0.054766617715358734, 0.05041288211941719, 0.05088745802640915, 0.04612748697400093, 0.04213482514023781, 0.038563504815101624, 0.041103772819042206, 0.03754572197794914, 0.0343821756541729, 0.031781043857336044, 0.027168739587068558, 0.025952372699975967, 0.02503393590450287, 0.0244587529450655, 0.020667126402258873, 0.020491890609264374, 0.020383255556225777, 0.02041975036263466 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.461771360794955e-06, "first_to_last_ratio": 4.607249847489048, "imbalance_abs_log_ratio": 1.5276311166276264, "last_quartile_mean": 1.6195716767697377e-06, "mean": 3.87299716564371e-06, "normalized": [ 3.0256593103213434, 2.4199948075671043, 2.2257260804728656, 1.8933484975335824, 1.6741432062616066, 1.4907581305464577, 1.3020525186737137, 1.38123110006882, 1.2950838303767327, 1.0716197853648062, 1.0585124142900662, 1.1105323556195759, 1.0395543790453834, 0.9297870003858463, 0.7930926998517134, 0.7672014355923814, 0.811551487187536, 0.7239809147787968, 0.6594474866461876, 0.6140918356353618, 0.649091321445497, 0.6059854562252807, 0.5735486080526876, 0.5386444043869645, 0.45665673160062437, 0.44875544656608446, 0.4444235140257583, 0.44115113302107534, 0.3820499025609541, 0.38571480138297476, 0.3910437033480733, 0.3955657011641444 ], "peak_layer": 1, "peak_normalized": 3.0256593103213434, "population_cv": 0.6453809227293102, "rest_mean": 4.185142588371322e-06, "spike_contrast": 0.5226612562717605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1874118829146026e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1718369933078066e-05, 9.372633030579891e-06, 8.620230801170692e-06, 7.332933364523342e-06, 6.483951892732875e-06, 5.773702014266746e-06, 5.042845714342548e-06, 5.349504135665484e-06, 5.015856004320085e-06, 4.150380391365616e-06, 4.099615580344107e-06, 4.30108866567025e-06, 4.026191163575277e-06, 3.60106241714675e-06, 3.0716457786184037e-06, 2.9713689855270786e-06, 3.1431366096512647e-06, 2.8039760309184203e-06, 2.5540382466715528e-06, 2.3783759388606995e-06, 2.5139288482023403e-06, 2.3469799543818226e-06, 2.2213521333469544e-06, 2.086168251480558e-06, 1.7686302271613386e-06, 1.7380285726176226e-06, 1.7212510101671796e-06, 1.7085770878111362e-06, 1.4796781897530309e-06, 1.4938723325030878e-06, 1.5145111547099077e-06, 1.5320248394345981e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00010837210174940992, "first_to_last_ratio": 4.663743740878985, "imbalance_abs_log_ratio": 1.5398185034733132, "last_quartile_mean": 2.3237147615873255e-05, "mean": 5.8729946545099665e-05, "normalized": [ 2.116910388942848, 2.12576419551759, 2.0713182282290723, 1.958808009785277, 1.85306829100999, 1.6774795633398605, 1.5342385858151337, 1.4245037147377415, 1.3659845454711594, 1.2527089446494675, 1.1583413114381589, 1.0816008878212584, 1.059050106445956, 0.9749946937544519, 0.9039297824077048, 0.8370924379536968, 0.8447837999101473, 0.7717212504681601, 0.708779947616627, 0.6523240772837903, 0.6962576204392495, 0.6381480261388907, 0.5854918741973255, 0.5414119569273494, 0.4665866617482466, 0.4484909063213352, 0.43561453714162146, 0.42649489708721455, 0.35024135478224894, 0.347602098795276, 0.34524713565577436, 0.34501016816737656 ], "peak_layer": 2, "peak_normalized": 2.12576419551759, "population_cv": 0.5682337234037166, "rest_mean": 6.323715206235647e-05, "spike_contrast": 0.5438422767363214, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4391036751912904e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001243260339833796, 0.00012484601757023484, 0.00012164840882178396, 0.00011504068970680237, 0.0001088306016754359, 9.851828508544713e-05, 9.010575013235211e-05, 8.366102701984346e-05, 8.022419933695346e-05, 7.357152935583144e-05, 6.802932330174372e-05, 6.352236232487485e-05, 6.219795614015311e-05, 5.726138624595478e-05, 5.308774780132808e-05, 4.916239413432777e-05, 4.961410741088912e-05, 4.5323147787712514e-05, 4.1626608435763046e-05, 3.8310958188958466e-05, 4.089117283001542e-05, 3.747839946299791e-05, 3.4385906474199146e-05, 3.1797095289221033e-05, 2.740260970313102e-05, 2.6339846954215318e-05, 2.558361848059576e-05, 2.5048022507689893e-05, 2.0569656044244766e-05, 2.041465268121101e-05, 2.02763458219124e-05, 2.026242873398587e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.029894954437623e-05, "first_to_last_ratio": 3.896020389480804, "imbalance_abs_log_ratio": 1.3599556192062257, "last_quartile_mean": 2.3177227149062674e-05, "mean": 5.08472848537167e-05, "normalized": [ 2.109310432443945, 2.0823500317454187, 1.994011612294058, 1.875066462699875, 1.727617067796283, 1.5815194849270788, 1.4631049290688447, 1.374102981258909, 1.3102874938711893, 1.2100951410093623, 1.1285247251248958, 1.0633171446519065, 1.0439918275480478, 0.9649933174114446, 0.9016610595399868, 0.8411183538760414, 0.8515897815206308, 0.7853791814165375, 0.7291252271433669, 0.6771443554330082, 0.737694287360078, 0.6786209457138218, 0.6314365240390994, 0.5913748367488056, 0.5313425485217884, 0.5201071760800584, 0.5115574693700321, 0.5058856398361036, 0.39753526518974286, 0.39544594503370456, 0.39314180453082337, 0.39154694679511176 ], "peak_layer": 1, "peak_normalized": 2.109310432443945, "population_cv": 0.5213289902477497, "rest_mean": 5.429271732282359e-05, "spike_contrast": 0.5938540399226916, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.224194952053949e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010725270840339363, 0.00010588184522930533, 0.00010139007645193487, 9.53420385485515e-05, 8.78446371643804e-05, 8.04159717517905e-05, 7.439491309924051e-05, 6.986940570641309e-05, 6.662456144113094e-05, 6.153005233500153e-05, 5.7382418162887916e-05, 5.406678974395618e-05, 5.3084149840287864e-05, 4.906729009235278e-05, 4.584701673593372e-05, 4.276858453522436e-05, 4.3301028199493885e-05, 3.993439895566553e-05, 3.707403811858967e-05, 3.4430951927788556e-05, 3.750975156435743e-05, 3.4506032534409314e-05, 3.210683280485682e-05, 3.006980477948673e-05, 2.7017325919587165e-05, 2.644603773660492e-05, 2.601130836410448e-05, 2.572291123215109e-05, 2.0213588868500665e-05, 2.0107352611375973e-05, 1.9990193322882988e-05, 1.9909099137294106e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002369592275499599, "first_to_last_ratio": 5.643806467094249, "imbalance_abs_log_ratio": 1.7305587433768272, "last_quartile_mean": 4.1985711050074315e-05, "mean": 0.00012327438730608264, "normalized": [ 2.0536938202390447, 2.174916037645403, 2.1925488795748866, 2.045836750341831, 2.0263612272059848, 1.7896425612539104, 1.6137064052293353, 1.480972211102483, 1.4347200865083345, 1.3015150111942828, 1.1747028946305826, 1.0857434163548976, 1.0627794297334374, 0.9758997849090596, 0.8960919751718553, 0.8230876844179131, 0.8258333502423459, 0.7456000475898096, 0.6776422362024666, 0.6227334054207839, 0.6492856625079976, 0.5954634942404196, 0.5397936868798249, 0.4867302228264425, 0.3971938000026409, 0.37230063475944564, 0.35633216718416544, 0.3460424570800059, 0.31457528168990173, 0.31217590063925915, 0.31141034981968424, 0.31466912740156466 ], "peak_layer": 3, "peak_normalized": 2.1925488795748866, "population_cv": 0.618597930641205, "rest_mean": 0.00013391951021427909, "spike_contrast": 0.49127064082412475, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.57907236018218e-05, "top_five_layers": [ 3, 2, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00025316784740425646, 0.00026811144198291004, 0.0002702851197682321, 0.00025219927192665637, 0.0002497984387446195, 0.00022061709023546427, 0.00019892866839654744, 0.00018256594194099307, 0.00017686423962004483, 0.0001604434655746445, 0.00014481077960226685, 0.000133844354422763, 0.00013101348304189742, 0.00012030344805680215, 0.00011046518920920789, 0.00010146562999580055, 0.00010180410026805475, 9.191338904201984e-05, 8.35359314805828e-05, 7.67670790082775e-05, 8.004029223229736e-05, 7.34053974156268e-05, 6.654273602180183e-05, 6.0001370002282783e-05, 4.896382233710028e-05, 4.5895132643636316e-05, 4.3926629587076604e-05, 4.265817187842913e-05, 3.877907511196099e-05, 3.848329288302921e-05, 3.838892007479444e-05, 3.8790643884567544e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04393457621335983, 0.024121688678860664, 0.030756350606679916, 0.03544071316719055, 0.03840360417962074, 0.026969486847519875, 0.027978016063570976, 0.029923241585493088, 0.03152087330818176, 0.024161184206604958, 0.025710709393024445, 0.02641460672020912, 0.02742084674537182, 0.021717891097068787, 0.0224718376994133, 0.023445574566721916, 0.024849263951182365, 0.01992032676935196, 0.02058842033147812, 0.020979218184947968, 0.02214338816702366, 0.01903437450528145, 0.02008410543203354, 0.021635910496115685, 0.023218298330903053, 0.022257626056671143, 0.024358727037906647, 0.025900650769472122, 0.027197737246751785, 0.02823900803923607, 0.02913021855056286, 0.029048049822449684 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.2108061784820165e-05, "first_to_last_ratio": 4.5038225950610284, "imbalance_abs_log_ratio": 1.5049265017537072, "last_quartile_mean": 4.908732819330908e-06, "mean": 1.424826272966584e-05, "normalized": [ 1.5393565072014856, 1.6436990565699428, 1.5746209905623882, 1.4393049982912232, 1.2296080351406573, 1.5542168455156227, 1.5210128581976592, 1.9112374884411145, 1.6656665742649008, 1.4395591772020961, 1.6324250537476421, 1.3835491754103626, 1.3477025444685555, 1.5824225341301699, 0.9384151980613074, 0.8636925984699546, 0.8034013225116797, 0.8830691126060858, 0.8605378947700126, 0.6634146810472719, 0.6587076476302224, 0.7136661453539468, 0.7077576665676224, 0.6868400571413836, 0.5793897205721035, 0.6202776091342075, 0.43119680892404477, 0.3692582531909216, 0.260043154103081, 0.12977534522113043, 0.15751431041740122, 0.20866063513380237 ], "peak_layer": 8, "peak_normalized": 1.9112374884411145, "population_cv": 0.5186010202163185, "rest_mean": 1.5120910120458575e-05, "spike_contrast": 0.6306476755313104, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.535966819385067e-06, "top_five_layers": [ 8, 9, 2, 11, 14 ], "top_five_spike_overlap": 0 }, "values": [ 2.1933155949227512e-05, 2.341985600651242e-05, 2.243561357317958e-05, 2.050759576377459e-05, 1.751977833919227e-05, 2.2144889953779057e-05, 2.167179081880022e-05, 2.723181387409568e-05, 2.3732854970148765e-05, 2.051121737167705e-05, 2.3259221052285284e-05, 1.9713172150659375e-05, 1.9202419935027137e-05, 2.2546772015630268e-05, 1.3370786291488912e-05, 1.2306119060667697e-05, 1.1447073120507412e-05, 1.2582200724864379e-05, 1.2261170013516676e-05, 9.452506674278993e-06, 9.385439625475556e-06, 1.0168502740270924e-05, 1.0084317182190716e-05, 9.786277587409131e-06, 8.255296961579006e-06, 8.837878340273164e-06, 6.143805421743309e-06, 5.26128860656172e-06, 3.705163180711679e-06, 1.8490732145437505e-06, 2.244305278509273e-06, 2.9730515507253585e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.226752566362848e-05, "first_to_last_ratio": 3.2558095109526493, "imbalance_abs_log_ratio": 1.1804409424216962, "last_quartile_mean": 1.9125051835544582e-05, "mean": 4.323053209986938e-05, "normalized": [ 1.0342606051842873, 1.898717126163711, 1.8089935381392723, 1.359356421079101, 1.274089744043777, 1.682360782557305, 1.2682690495969076, 1.1968320759706532, 1.2588398746694436, 1.2880627600055157, 1.2248689931301155, 1.273330768170305, 1.3149263319378108, 1.3464942246682916, 1.1005868831933079, 1.0691381511055633, 1.0591169598492518, 1.0073228103092708, 0.9327165506900205, 0.7338249833201036, 0.7729257473598179, 0.8419776567654331, 0.8893297062436448, 0.8244833428272487, 0.7679470877937514, 0.8196268734117844, 0.5402279437877997, 0.47239671566976066, 0.3242697700833044, 0.15928933562535502, 0.20350298326020644, 0.25191420338787907 ], "peak_layer": 2, "peak_normalized": 1.898717126163711, "population_cv": 0.42885792050879057, "rest_mean": 4.467689194387748e-05, "spike_contrast": 0.7928078118486851, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.542018894222565e-05, "top_five_layers": [ 2, 3, 6, 4, 14 ], "top_five_spike_overlap": 0 }, "values": [ 4.471163629204966e-05, 8.208255167119205e-05, 7.82037532189861e-05, 5.8765701396623626e-05, 5.507957757799886e-05, 7.272935181390494e-05, 5.482794585986994e-05, 5.173968747840263e-05, 5.442031761049293e-05, 5.568363849306479e-05, 5.295173832564615e-05, 5.50467666471377e-05, 5.684496500180103e-05, 5.820966180181131e-05, 4.757895658258349e-05, 4.621941116056405e-05, 4.578618973027915e-05, 4.354710108600557e-05, 4.0321832784684375e-05, 3.172364449710585e-05, 3.341399133205414e-05, 3.639914211817086e-05, 3.8446196413133293e-05, 3.564285361790098e-05, 3.319876122986898e-05, 3.543290586094372e-05, 2.3354341465164907e-05, 2.0421961380634457e-05, 1.4018354704603553e-05, 6.886162736918777e-06, 8.797542250249535e-06, 1.0890385055972729e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05521946307271719, "first_to_last_ratio": 3.255809416547513, "imbalance_abs_log_ratio": 1.180440913425793, "last_quartile_mean": 0.016960287292022258, "mean": 0.03833726760058198, "normalized": [ 1.0342606436254083, 1.89871698336364, 1.808993472457683, 1.3593564915846248, 1.2740897521241208, 1.682360722725715, 1.2682690813216775, 1.1968320831111716, 1.2588398492110833, 1.2880628213558492, 1.224868979767805, 1.273330745435787, 1.314926371992558, 1.3464941899184253, 1.1005869061077207, 1.069138220456837, 1.059116922317307, 1.0073228539527392, 0.932716509673011, 0.7338250308305378, 0.7729257302043212, 0.841977653627328, 0.8893297206164247, 0.8244832831061564, 0.7679470991481019, 0.8196269423990945, 0.5402279418758984, 0.4723966869996653, 0.32426976863400875, 0.15928933802210024, 0.20350299592243773, 0.25191420811076204 ], "peak_layer": 2, "peak_normalized": 1.89871698336364, "population_cv": 0.42885790694027637, "rest_mean": 0.039619914031829, "spike_contrast": 0.7928077998002174, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03141097687184811, "top_five_layers": [ 2, 3, 6, 4, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.039650727063417435, 0.07279162108898163, 0.06935186684131622, 0.05211401358246803, 0.04884511977434158, 0.06449711322784424, 0.048621971160173416, 0.04588327184319496, 0.04826048016548157, 0.04938080906867981, 0.04695812985301018, 0.0488160215318203, 0.0504106841981411, 0.051620908081531525, 0.042193494737148285, 0.04098783805966377, 0.040603648871183395, 0.03861800581216812, 0.03575780242681503, 0.02813284657895565, 0.029631860554218292, 0.032279122620821, 0.034094471484422684, 0.03160843625664711, 0.029440993443131447, 0.03142225742340088, 0.020710863173007965, 0.018110398203134537, 0.012431616894900799, 0.006106717977672815, 0.007801748812198639, 0.009657702408730984 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2073289212821692e-05, "first_to_last_ratio": 5.118173394564018, "imbalance_abs_log_ratio": 1.6327976165236144, "last_quartile_mean": 2.358905859978222e-06, "mean": 9.608797252980139e-06, "normalized": [ 2.2116042312522914, 1.2252124207577697, 0.9166421564982928, 1.186820314517719, 0.9292558997947784, 1.1346790160633606, 1.0583515009014062, 1.3892974727904446, 1.5862793081829674, 1.2655275776533215, 1.4193695477429429, 2.042377666614073, 1.9979815760663124, 2.116845169548275, 1.4502152158936208, 1.0590931016242418, 1.4220084536973343, 1.379051361973986, 1.173597294405293, 0.9241153337121745, 0.6121286422299292, 0.5710108338819657, 0.461893587104583, 0.5026871640197357, 0.30513715456990576, 0.2951723976971137, 0.2866930192727529, 0.2870900856457325, 0.21497083513437823, 0.17484004907583897, 0.18455905309901366, 0.21549255857844585 ], "peak_layer": 1, "peak_normalized": 2.2116042312522914, "population_cv": 0.6038429581123452, "rest_mean": 1.0515277882337546e-05, "spike_contrast": 0.44828124441370076, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.713801854450139e-06, "top_five_layers": [ 1, 14, 12, 13, 9 ], "top_five_spike_overlap": 0 }, "values": [ 2.1250856661936268e-05, 1.1772817742894404e-05, 8.807828635326587e-06, 1.1403915777918883e-05, 8.929031537263654e-06, 1.0902900612563826e-05, 1.0169484994548839e-05, 1.3349477740121074e-05, 1.5242236258927733e-05, 1.2160197911725845e-05, 1.3638434211316053e-05, 1.9624792912509292e-05, 1.919819987961091e-05, 2.034033605013974e-05, 1.3934823982708622e-05, 1.017661088553723e-05, 1.3663790923601482e-05, 1.3251024938654155e-05, 1.1276858458586503e-05, 8.879636880010366e-06, 5.8818200159294065e-06, 5.4867273320269305e-06, 4.43824183093966e-06, 4.830219040741213e-06, 2.9320010526134865e-06, 2.836251724147587e-06, 2.75477509603661e-06, 2.7585904263105476e-06, 2.06561117011006e-06, 1.6800025832708343e-06, 1.773390522430418e-06, 2.0706243049062323e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.736539585792343e-05, "first_to_last_ratio": 5.693189725390274, "imbalance_abs_log_ratio": 1.7392706754966003, "last_quartile_mean": 8.319658775235439e-06, "mean": 3.0428698622131378e-05, "normalized": [ 0.9332234373969145, 2.093244562439342, 2.000219271237334, 1.4660610383042352, 1.4267640458493276, 1.8877254776078727, 1.397617962458032, 1.2479661794130616, 1.4262249611304942, 1.4477791439571595, 1.24181841376056, 1.2462360629867504, 1.2738256274260629, 1.4269714780931013, 1.1746071387256543, 1.0297217541028905, 1.0028420024769986, 1.0837416535661664, 0.9989109522318798, 0.7645609375506638, 0.8126373951104926, 0.8905222845518224, 0.8066853450678224, 0.7327738890015353, 0.50846646429871, 0.4349609059082798, 0.29076653368375877, 0.2846228030965179, 0.21016872042077955, 0.130190170117833, 0.14875731368946996, 0.17938607433847711 ], "peak_layer": 2, "peak_normalized": 2.093244562439342, "population_cv": 0.5294278422744164, "rest_mean": 3.183621146052718e-05, "spike_contrast": 0.7170491791430023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.2828129294794054e-05, "top_five_layers": [ 2, 3, 6, 4, 10 ], "top_five_spike_overlap": 0 }, "values": [ 2.83967747236602e-05, 6.369470793288201e-05, 6.086406938266009e-05, 4.461032949620858e-05, 4.3414573156042024e-05, 5.744102963944897e-05, 4.2527695768512785e-05, 3.797398676397279e-05, 4.339816950960085e-05, 4.405403524287976e-05, 3.7786918255733326e-05, 3.7921341572655365e-05, 3.876085611409508e-05, 4.342088504927233e-05, 3.5741766623687e-05, 3.133309292024933e-05, 3.051517705898732e-05, 3.297684816061519e-05, 3.0395560315810144e-05, 2.3264594346983358e-05, 2.4727498384891078e-05, 2.7097434212919325e-05, 2.4546385247958824e-05, 2.2297355826594867e-05, 1.547197280160617e-05, 1.323529431829229e-05, 8.847647222864907e-06, 8.660701496410184e-06, 6.39516065348289e-06, 3.961517450079555e-06, 4.52649146609474e-06, 5.458484793052776e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.615552941482747e-05, "first_to_last_ratio": 8.764492263828256, "imbalance_abs_log_ratio": 2.1707085890088735, "last_quartile_mean": 4.1252280595927004e-06, "mean": 2.208462114339227e-05, "normalized": [ 0.7494933219758998, 2.2634292925463795, 2.1410570182439597, 1.5739498957546751, 1.5487374547204664, 2.0376524711677733, 1.534141636657325, 1.2486273054515569, 1.5321016320199576, 1.6393386005987565, 1.2261935965091952, 1.1547399796224558, 1.219246066181087, 1.4546728804264264, 1.2279261343623535, 1.0312465593605784, 0.9063241145323925, 1.1172138768757267, 1.0304062764426833, 0.7847973944776799, 0.8360866099062609, 0.911401228786939, 0.6978180640030948, 0.6390634924568735, 0.31508360869438684, 0.26042370730894115, 0.18982145136915554, 0.23533143612154772, 0.14063662904795424, 0.10306654364310344, 0.11411045319594963, 0.13586126753846497 ], "peak_layer": 2, "peak_normalized": 2.2634292925463795, "population_cv": 0.6059056054769965, "rest_mean": 2.3393786848289352e-05, "spike_contrast": 0.6418424872519509, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.5015126336948015e-05, "top_five_layers": [ 2, 3, 6, 10, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.6552276065340266e-05, 4.998697841074318e-05, 4.7284433094318956e-05, 3.476008714642376e-05, 3.420327993808314e-05, 4.500078284763731e-05, 3.388093682588078e-05, 2.757546099019237e-05, 3.383588409633376e-05, 3.620417191996239e-05, 2.708002102735918e-05, 2.5501994969090447e-05, 2.6926587452180684e-05, 3.212589945178479e-05, 2.7118283469462767e-05, 2.277468956890516e-05, 2.0015824702568352e-05, 2.467324520694092e-05, 2.2756132239010185e-05, 1.7331953131360933e-05, 1.8464656022842973e-05, 2.0127950847381726e-05, 1.5411047570523806e-05, 1.4113475117483176e-05, 6.958502126508392e-06, 5.7513589126756415e-06, 4.19213483837666e-06, 5.1972056098748e-06, 3.1059066714078654e-06, 2.276185568916844e-06, 2.520086127333343e-06, 3.0004446216480574e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011569616435735952, "first_to_last_ratio": 4.483785350026755, "imbalance_abs_log_ratio": 1.500467633864892, "last_quartile_mean": 2.5803234393606544e-05, "mean": 7.895687571135568e-05, "normalized": [ 1.113643291734891, 1.9315033327137208, 1.8689084538217502, 1.3013119102609747, 1.3204401608174607, 1.7172337521094045, 1.2105960380108525, 1.2588293628008012, 1.3316624978487503, 1.2436188212370092, 1.3620564885728002, 1.3794695319626817, 1.4052872102652663, 1.3953280795865155, 1.1319795963186805, 0.9796315470123751, 1.1579414909870167, 1.0808582581724016, 0.9982944348550936, 0.7435043404975695, 0.8011357344053238, 0.8526183967497576, 0.9599446768973036, 0.8397896829211625, 0.7011434784757186, 0.5094988193534132, 0.32142473628464113, 0.2742399520171878, 0.26163364470536055, 0.152278590465612, 0.18046309892286552, 0.21373058921563795 ], "peak_layer": 2, "peak_normalized": 1.9315033327137208, "population_cv": 0.47743629272692334, "rest_mean": 8.142900973056115e-05, "spike_contrast": 0.8056999860950416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.560735200764612e-05, "top_five_layers": [ 2, 3, 6, 13, 14 ], "top_five_spike_overlap": 0 }, "values": [ 8.79297949722968e-05, 0.00015250546857714653, 0.00014756317250430584, 0.00010274752276018262, 0.00010425782966194674, 0.00013558741193264723, 9.558488090988249e-05, 9.939323354046792e-05, 0.00010514391033211723, 9.819225670071319e-05, 0.00010754372488008812, 0.00010891860438277945, 0.00011095708759967238, 0.00011017074575647712, 8.937757229432464e-05, 7.734864630037919e-05, 9.142744238488376e-05, 8.53411911521107e-05, 7.882220961619169e-05, 5.870477980352007e-05, 6.32551746093668e-05, 6.732008478138596e-05, 7.579423254355788e-05, 6.630716961808503e-05, 5.5360098485834897e-05, 4.022843495476991e-05, 2.5378692953381687e-05, 2.165312980650924e-05, 2.0657775166910142e-05, 1.2023441740893759e-05, 1.4248802472138777e-05, 1.6875499568413943e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.018067941069602966, 0.027584951370954514, 0.03231721371412277, 0.036101218312978745, 0.025345133617520332, 0.027166159823536873, 0.028526438400149345, 0.02979987859725952, 0.02248864620923996, 0.023894935846328735, 0.02451697364449501, 0.025183402001857758, 0.020499059930443764, 0.02105838619172573, 0.0216353852301836, 0.022715488448739052, 0.01881815865635872, 0.0193756353110075, 0.02001863345503807, 0.021088071167469025, 0.018417155370116234, 0.019219009205698967, 0.020342160016298294, 0.02150510996580124, 0.02099662832915783, 0.02211105264723301, 0.02328667603433132, 0.024160878732800484, 0.02260657213628292, 0.024989914149045944, 0.02536177635192871, 0.027097374200820923 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.2429669027369528e-05, "first_to_last_ratio": 1.5167074865147177, "imbalance_abs_log_ratio": 0.4165418581180973, "last_quartile_mean": 8.195165605684451e-06, "mean": 1.1438181132916725e-05, "normalized": [ 2.3239055235296235, 1.1190114244144547, 0.8331917749372405, 0.7813452753113945, 1.003803456769089, 0.9365675136194498, 0.8842494964053754, 0.8113839672107822, 1.0847409217961617, 1.0682117256400467, 0.9883998597662562, 0.9038042770049443, 1.1938503302540149, 1.1817190494107024, 1.0722151723600664, 0.9843285081524176, 1.2863323428556752, 1.156982982287669, 1.0862279117485159, 1.0234351273362494, 1.2841852282798012, 1.1754321217296801, 1.070502203826642, 1.0143774573484126, 0.8953924191337335, 0.8270637955966683, 0.8133202903656793, 0.7381861493144678, 0.6730532700250402, 0.5804717059865397, 0.6082410306470007, 0.5960676869362054 ], "peak_layer": 1, "peak_normalized": 2.3239055235296235, "population_cv": 0.3045451785807339, "rest_mean": 1.1251827985419098e-05, "spike_contrast": 1.1059970118215763, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2444488129403908e-05, "top_five_layers": [ 1, 17, 21, 13, 14 ], "top_five_spike_overlap": 1 }, "values": [ 2.6581252313917503e-05, 1.2799455362255685e-05, 9.530198440188542e-06, 8.937168786360417e-06, 1.1481685760372784e-05, 1.0712628863984719e-05, 1.011420590657508e-05, 9.280756785301492e-06, 1.2407463145791553e-05, 1.22183992061764e-05, 1.1305496627755929e-05, 1.0337877029087394e-05, 1.3655476323037874e-05, 1.3516716535377782e-05, 1.2264191354915965e-05, 1.125892777054105e-05, 1.4713302334712353e-05, 1.323378091910854e-05, 1.2424471606209408e-05, 1.1706236364261713e-05, 1.4688743249280378e-05, 1.3444805517792702e-05, 1.2244598110555671e-05, 1.1602633094298653e-05, 1.0241660675092135e-05, 9.460105502512306e-06, 9.302904800279066e-06, 8.443506885669194e-06, 7.698505214648321e-06, 6.639540515607223e-06, 6.957171081012348e-06, 6.817930170655018e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.8672892262402456e-05, "first_to_last_ratio": 1.4790455122350374, "imbalance_abs_log_ratio": 0.39139695555746173, "last_quartile_mean": 3.290831273261574e-05, "mean": 4.5425740495375067e-05, "normalized": [ 2.292867781479936, 1.098844858197334, 0.8288263395703189, 0.7584336983627002, 0.9778795698933019, 0.9351432056276012, 0.8751120479081964, 0.8047536035443427, 1.088349470440326, 1.0783459726467945, 0.9909402014503932, 0.9019584923345375, 1.2027204540239667, 1.175772139842897, 1.0782465054806212, 0.9953149147774686, 1.2799515830354413, 1.1648392407100392, 1.0907985090406367, 1.0337452797487645, 1.2844092657539465, 1.1758215530808431, 1.0768851187191208, 1.0145044696518641, 0.9072210422189393, 0.8359231056438284, 0.8227296910259355, 0.7472276249111484, 0.6761454407882689, 0.5893866866897062, 0.6130538257244701, 0.6038483076763113 ], "peak_layer": 1, "peak_normalized": 2.292867781479936, "population_cv": 0.29963351102833546, "rest_mean": 4.465376965796437e-05, "spike_contrast": 1.1106426937137044, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.9594383017392826e-05, "top_five_layers": [ 1, 21, 17, 13, 22 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010415521683171391, 4.9915841373149306e-05, 3.765005021705292e-05, 3.445241236477159e-05, 4.442090357770212e-05, 4.2479572584852576e-05, 3.975261279265396e-05, 3.655652835732326e-05, 4.943908061250113e-05, 4.898466431768611e-05, 4.501419243752025e-05, 4.097213241038844e-05, 5.4634467232972383e-05, 5.3410320106195286e-05, 4.89801459480077e-05, 4.5212917029857635e-05, 5.814274845761247e-05, 5.291368506732397e-05, 4.9550330004421994e-05, 4.695864481618628e-05, 5.8345241995994e-05, 5.341256473911926e-05, 4.891830394626595e-05, 4.608461676980369e-05, 4.1211187635781243e-05, 3.797242607106455e-05, 3.737310544238426e-05, 3.394336818018928e-05, 3.071440733037889e-05, 2.6773326680995524e-05, 2.784842399705667e-05, 2.7430256523075514e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0431636068969965, "first_to_last_ratio": 1.4790455355871048, "imbalance_abs_log_ratio": 0.39139697134606793, "last_quartile_mean": 0.02918341988697648, "mean": 0.04028400045353919, "normalized": [ 2.2928676956325664, 1.0988448461233453, 0.8288263129280055, 0.7584337335279033, 0.9778795372444193, 0.9351432892874852, 0.8751120548755073, 0.8047535990009899, 1.0883495038762179, 1.0783460397661777, 0.9909401548295442, 0.9019584843211922, 1.2027204645548106, 1.1757721269107668, 1.0782465359354751, 0.9953149017803241, 1.2799516204237265, 1.1648392822258575, 1.090798537285597, 1.0337452971674015, 1.2844092255829724, 1.1758215089234016, 1.0768851089878908, 1.014504529948594, 0.9072209977912502, 0.8359230814928427, 0.8227296688325424, 0.7472276232112787, 0.676145406434352, 0.5893866663000006, 0.6130538743742001, 0.6038482904233612 ], "peak_layer": 1, "peak_normalized": 2.2928676956325664, "population_cv": 0.29963350187956755, "rest_mean": 0.0395994091199504, "spike_contrast": 1.110642674533265, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04398079365491867, "top_five_layers": [ 1, 21, 17, 13, 22 ], "top_five_spike_overlap": 2 }, "values": [ 0.09236588329076767, 0.04426586627960205, 0.03338843956589699, 0.03055274486541748, 0.039392899721860886, 0.03767131268978119, 0.03525301441550255, 0.03241869434714317, 0.04384307190775871, 0.043440092355012894, 0.03991903364658356, 0.03633449599146843, 0.04845039173960686, 0.04736480489373207, 0.04343608394265175, 0.040095265954732895, 0.05156157165765762, 0.04692438617348671, 0.04394172877073288, 0.04164339601993561, 0.05174114182591438, 0.04736679419875145, 0.04338124021887779, 0.04086830094456673, 0.036546491086483, 0.03367432579398155, 0.03314284235239029, 0.030101317912340164, 0.027237841859459877, 0.023742852732539177, 0.024696262553334236, 0.024325424805283546 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.470277960104795e-06, "first_to_last_ratio": 1.5769587704873496, "imbalance_abs_log_ratio": 0.4554981633698566, "last_quartile_mean": 2.2006142614827695e-06, "mean": 2.887214563429552e-06, "normalized": [ 2.7519535335351715, 1.313168413337379, 0.9399430071407856, 0.8753790372729023, 1.0813834937551676, 0.9567319703631147, 0.8229162441441861, 0.8740977436183472, 1.0566599601043785, 0.9454608388064315, 0.9224389765316489, 0.8092920053718847, 1.0583487949182648, 1.0534209725986503, 0.9874597862575576, 0.8539122103086818, 1.169801497382018, 1.081662118029895, 1.0127159978603695, 0.961060648114576, 1.1999729309980693, 1.1228483457198037, 1.0226342502719021, 1.0291945210343731, 0.8967756962809945, 0.8510820239929227, 0.8261498765197046, 0.7721004685625728, 0.7451436093586472, 0.6285081646269207, 0.6879375096468019, 0.6898453535358771 ], "peak_layer": 1, "peak_normalized": 2.7519535335351715, "population_cv": 0.35016697743385605, "rest_mean": 2.8581899574352e-06, "spike_contrast": 1.0649912990844543, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.043947435799055e-06, "top_five_layers": [ 1, 2, 21, 17, 22 ], "top_five_spike_overlap": 2 }, "values": [ 7.945480319904163e-06, 3.7913989672233583e-06, 2.7138171390106436e-06, 2.5274071049352642e-06, 3.12218617182225e-06, 2.7622904781310353e-06, 2.3759357645758428e-06, 2.523707735235803e-06, 3.0508040254062507e-06, 2.729748302954249e-06, 2.6632792469172273e-06, 2.3365996639768127e-06, 3.055680053876131e-06, 3.041452373508946e-06, 2.851008275683853e-06, 2.4654277694935445e-06, 3.3774679195630597e-06, 3.122990619885968e-06, 2.92392837764055e-06, 2.774788299575448e-06, 3.4645793220988708e-06, 3.2419040962849976e-06, 2.952564500446897e-06, 2.9715054097323446e-06, 2.589183850432164e-06, 2.4572564143454656e-06, 2.3852719550632173e-06, 2.229219717264641e-06, 2.1513894807867473e-06, 1.814637926145224e-06, 1.9862231965817045e-06, 1.9917315512429923e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.1570810026314575e-05, "first_to_last_ratio": 1.5044497617163743, "imbalance_abs_log_ratio": 0.40842722451739255, "last_quartile_mean": 2.763190309451602e-05, "mean": 3.7813772848949156e-05, "normalized": [ 2.388009071075982, 1.1470596649494917, 0.8534302761319221, 0.7776431816025992, 0.9983567968228081, 0.9421045940312636, 0.8787540856715802, 0.8094928604716568, 1.0777400416324399, 1.0654722038935143, 0.9718539610468895, 0.8865810688989143, 1.168850821337844, 1.1332939684204937, 1.0417735360252487, 0.9621498679953465, 1.2282401671332226, 1.126146404473492, 1.059853477638917, 1.0196054375892694, 1.2877065753533457, 1.1949291884271547, 1.0946140188339069, 1.0404469270679417, 0.9334039480335107, 0.8520038997158634, 0.8392353967355263, 0.7615998622978171, 0.6682831346061853, 0.5860512257947129, 0.6091233405529298, 0.5961909957382107 ], "peak_layer": 1, "peak_normalized": 2.388009071075982, "population_cv": 0.309055923286973, "rest_mean": 3.704195080866123e-05, "spike_contrast": 1.13335315635395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.198161186650395e-05, "top_five_layers": [ 1, 21, 17, 22, 13 ], "top_five_spike_overlap": 2 }, "values": [ 9.029963257489726e-05, 4.337465361459181e-05, 3.227141860406846e-05, 2.9405622626654804e-05, 3.775163713726215e-05, 3.562452911864966e-05, 3.322900738567114e-05, 3.060997914872132e-05, 4.0753417124506086e-05, 4.0289523894898593e-05, 3.674946492537856e-05, 3.3524975151522085e-05, 4.419865945237689e-05, 4.2854120692936704e-05, 3.9393387851305306e-05, 3.638251655502245e-05, 4.644439468393102e-05, 4.258384433342144e-05, 4.007705865660682e-05, 3.855512841255404e-05, 4.869304393650964e-05, 4.518478090176359e-05, 4.139148586546071e-05, 3.934322376153432e-05, 3.5295524867251515e-05, 3.2217481930274516e-05, 3.173465665895492e-05, 2.879896419472061e-05, 2.5270306650782004e-05, 2.2160807930049486e-05, 2.303325163666159e-05, 2.2544230887433514e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.563517248039716e-05, "first_to_last_ratio": 1.4352355648633195, "imbalance_abs_log_ratio": 0.3613289924385252, "last_quartile_mean": 2.4828796995279845e-05, "mean": 3.227010705586508e-05, "normalized": [ 2.4014447290909864, 1.1673853090110469, 0.8593097269193264, 0.7832036205871891, 1.0067937101766118, 0.9335226700834045, 0.8775072266073977, 0.8050578010660164, 1.053006259394381, 1.0264307263592312, 0.9338631869266799, 0.8595966381479759, 1.1212921464302679, 1.0758485643993823, 0.9942738962286088, 0.9189156755903263, 1.1746749961886236, 1.0800838025282244, 1.019430881526982, 0.9950392558872734, 1.2898668670070823, 1.2311059818213306, 1.13592608464973, 1.1011764563471198, 0.9942343261534591, 0.8955937948027416, 0.9014365249588188, 0.831025803802073, 0.6870790019632969, 0.6046383015495036, 0.6312332814164576, 0.610002752378451 ], "peak_layer": 1, "peak_normalized": 2.4014447290909864, "population_cv": 0.30444570497962015, "rest_mean": 3.1370954645724315e-05, "spike_contrast": 1.1834364140297289, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.7125530070625246e-05, "top_five_layers": [ 1, 21, 22, 17, 2 ], "top_five_spike_overlap": 2 }, "values": [ 7.749487849650905e-05, 3.7671648897230625e-05, 2.7730016881832853e-05, 2.527406468288973e-05, 3.2489340810570866e-05, 3.0124876502668485e-05, 2.8317252144915983e-05, 2.5979301426559687e-05, 3.398062472115271e-05, 3.312302942504175e-05, 3.0135865017655306e-05, 2.77392755378969e-05, 3.618421760620549e-05, 3.4717748349066824e-05, 3.2085325074149296e-05, 2.965350722661242e-05, 3.790688788285479e-05, 3.4854419936891645e-05, 3.289714368293062e-05, 3.211002331227064e-05, 4.162414188613184e-05, 3.972792183049023e-05, 3.6656456359196454e-05, 3.55350821337197e-05, 3.2084048143588006e-05, 2.8900907636852935e-05, 2.9089453164488077e-05, 2.6817291654879227e-05, 2.2172112949192524e-05, 1.9511742721078917e-05, 2.03699655685341e-05, 1.9684854123624973e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.83701450220542e-05, "first_to_last_ratio": 1.5137756203780361, "imbalance_abs_log_ratio": 0.41460694084716315, "last_quartile_mean": 5.837730759594706e-05, "mean": 8.360470872048609e-05, "normalized": [ 2.260874919572962, 1.046876661671232, 0.8108813008392315, 0.7410270778432692, 0.9650502462422242, 0.9363082641888973, 0.8839048255705971, 0.8110736329048943, 1.115418327104151, 1.1154856868411749, 1.0274711394986145, 0.9306550068796721, 1.2562444747790198, 1.2247146350825069, 1.1283567053258787, 1.0398138975133757, 1.305825331548614, 1.176867029634062, 1.1082958616806637, 1.034638772497703, 1.2882363465735436, 1.1524683581694375, 1.0651417678309785, 0.9883391355795494, 0.8670300800809992, 0.8013878422949675, 0.7767996236633684, 0.7006926068874338, 0.6674164616571702, 0.5747719332171776, 0.5954528519609661, 0.6024791948656639 ], "peak_layer": 1, "peak_normalized": 2.260874919572962, "population_cv": 0.30484696934594563, "rest_mean": 8.24862149278892e-05, "spike_contrast": 1.0867825039478176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.964457520050928e-05, "top_five_layers": [ 1, 17, 21, 13, 14 ], "top_five_spike_overlap": 1 }, "values": [ 0.0001890197891043499, 8.752381836529821e-05, 6.77934949635528e-05, 6.195335299707949e-05, 8.068274473771453e-05, 7.82797797000967e-05, 7.389860547846183e-05, 6.780957482988015e-05, 9.325422433903441e-05, 9.325985593022779e-05, 8.590142533648759e-05, 7.780714076943696e-05, 0.00010502795339561999, 0.0001023919103317894, 9.433593368157744e-05, 8.693333802511916e-05, 0.00010917314648395404, 9.839162521529943e-05, 9.265875269193202e-05, 8.650067320559174e-05, 0.00010770262451842427, 9.635178139433265e-05, 8.905086724553257e-05, 8.262980554718524e-05, 7.248779729707167e-05, 6.69997971272096e-05, 6.494410627055913e-05, 5.858120130142197e-05, 5.579915887210518e-05, 4.8053640057332814e-05, 4.978266224497929e-05, 5.037009759689681e-05 ] } } } } } }, "step": 500 }, { "modes": { "learned": { "bits_per_byte": 2.073383110809672, "forward": { "activation_sha256": { "attention_branch_output": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", "mlp_branch_output": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", "post_attention_state": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", "post_mlp_state": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", "pre_attention_input": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", "pre_mlp_input": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93" }, "logits_sha256": "9adeddfaa352b7946853a2fe6e55c7269e6a01f6272498cdfb911b26410503c6", "loss_nats": 1.4371596574783325, "mixer_summary_sha256": "739af9a4ced9bd8d3e1cd2555b418bfc5db56ce4da6de7d1c45506ae68b1e3e1" }, "loss_nats": 1.4371596574783325, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.4738388657569885, "group": 1, "latest_source_mass": 0.19323334097862244, "layer": 1, "max_source_mass": 0.8067666888237, "mixer_index": 1, "normalized_entropy": 0.6836049818080586, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.8067666888237, "median": 0.8265105485916138, "p05": 0.6604017019271851, "p95": 0.8949732184410095 }, { "label": "current_group_partial", "mean": 0.19323334097862244, "median": 0.17348942160606384, "p05": 0.10502681136131287, "p95": 0.33959829807281494 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6344536542892456, "group": 1, "latest_source_mass": 0.5104025602340698, "layer": 2, "max_source_mass": 0.5104025602340698, "mixer_index": 2, "normalized_entropy": 0.9153231407169755, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4895974397659302, "median": 0.45956116914749146, "p05": 0.2515213191509247, "p95": 0.7871550917625427 }, { "label": "current_group_partial", "mean": 0.5104025602340698, "median": 0.5404388308525085, "p05": 0.21284496784210205, "p95": 0.7484787106513977 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.41773349046707153, "group": 1, "latest_source_mass": 0.15382957458496094, "layer": 2, "max_source_mass": 0.8461704254150391, "mixer_index": 3, "normalized_entropy": 0.6026620351100812, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8461704254150391, "median": 0.8643104434013367, "p05": 0.7285550832748413, "p95": 0.9108630418777466 }, { "label": "current_group_partial", "mean": 0.15382957458496094, "median": 0.13568954169750214, "p05": 0.08913692831993103, "p95": 0.2714449167251587 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6868985891342163, "group": 1, "latest_source_mass": 0.49918031692504883, "layer": 3, "max_source_mass": 0.500819742679596, "mixer_index": 4, "normalized_entropy": 0.9909851881375595, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.500819742679596, "median": 0.50103759765625, "p05": 0.41182032227516174, "p95": 0.5924672484397888 }, { "label": "current_group_partial", "mean": 0.49918031692504883, "median": 0.49896240234375, "p05": 0.40753278136253357, "p95": 0.5881797075271606 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5841774344444275, "group": 1, "latest_source_mass": 0.315071702003479, "layer": 3, "max_source_mass": 0.684928297996521, "mixer_index": 5, "normalized_entropy": 0.8427898876722131, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.684928297996521, "median": 0.6976089477539062, "p05": 0.4487886428833008, "p95": 0.8860265612602234 }, { "label": "current_group_partial", "mean": 0.315071702003479, "median": 0.3023911118507385, "p05": 0.1139734610915184, "p95": 0.5512113571166992 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6896515488624573, "group": 1, "latest_source_mass": 0.49074798822402954, "layer": 4, "max_source_mass": 0.5092520117759705, "mixer_index": 6, "normalized_entropy": 0.9949568694852597, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5092520117759705, "median": 0.509276270866394, "p05": 0.4409506916999817, "p95": 0.5760948657989502 }, { "label": "current_group_partial", "mean": 0.49074798822402954, "median": 0.49072372913360596, "p05": 0.4239051342010498, "p95": 0.5590493679046631 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5611414909362793, "group": 1, "latest_source_mass": 0.2719496786594391, "layer": 4, "max_source_mass": 0.7280503511428833, "mixer_index": 7, "normalized_entropy": 0.8095560462108093, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7280503511428833, "median": 0.7264741063117981, "p05": 0.5577248930931091, "p95": 0.8873021602630615 }, { "label": "current_group_partial", "mean": 0.2719496786594391, "median": 0.2735258936882019, "p05": 0.11269781738519669, "p95": 0.44227516651153564 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6817618608474731, "group": 2, "latest_source_mass": 0.4686949551105499, "layer": 5, "max_source_mass": 0.5313050746917725, "mixer_index": 8, "normalized_entropy": 0.9835744557118811, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5313050746917725, "median": 0.532880961894989, "p05": 0.4224134683609009, "p95": 0.6363531351089478 }, { "label": "completed_group_1", "mean": 0.4686949551105499, "median": 0.4671190679073334, "p05": 0.363646924495697, "p95": 0.5775865316390991 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0218309164047241, "group": 2, "latest_source_mass": 0.2602040767669678, "layer": 5, "max_source_mass": 0.4974328279495239, "mixer_index": 9, "normalized_entropy": 0.9301105830916285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4974328279495239, "median": 0.4930803179740906, "p05": 0.35206541419029236, "p95": 0.6587448716163635 }, { "label": "completed_group_1", "mean": 0.2423631250858307, "median": 0.23208677768707275, "p05": 0.15309149026870728, "p95": 0.3585616946220398 }, { "label": "current_group_partial", "mean": 0.2602040767669678, "median": 0.261092871427536, "p05": 0.1870650351047516, "p95": 0.33141374588012695 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0932960510253906, "group": 2, "latest_source_mass": 0.34662574529647827, "layer": 6, "max_source_mass": 0.3533782958984375, "mixer_index": 10, "normalized_entropy": 0.9951609519595268, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3533782958984375, "median": 0.3537653684616089, "p05": 0.300750195980072, "p95": 0.40760302543640137 }, { "label": "completed_group_1", "mean": 0.29999595880508423, "median": 0.2998189330101013, "p05": 0.277343213558197, "p95": 0.3256012797355652 }, { "label": "current_group_partial", "mean": 0.34662574529647827, "median": 0.34948140382766724, "p05": 0.3000357747077942, "p95": 0.3795154392719269 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0456881523132324, "group": 2, "latest_source_mass": 0.3350909352302551, "layer": 6, "max_source_mass": 0.4338536858558655, "mixer_index": 11, "normalized_entropy": 0.9518263750544431, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4338536858558655, "median": 0.43320339918136597, "p05": 0.2745198607444763, "p95": 0.6123260259628296 }, { "label": "completed_group_1", "mean": 0.23105540871620178, "median": 0.22906750440597534, "p05": 0.15816110372543335, "p95": 0.3054558038711548 }, { "label": "current_group_partial", "mean": 0.3350909352302551, "median": 0.33941850066185, "p05": 0.2199106216430664, "p95": 0.4273326098918915 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0891339778900146, "group": 2, "latest_source_mass": 0.37785181403160095, "layer": 7, "max_source_mass": 0.37785181403160095, "mixer_index": 12, "normalized_entropy": 0.9913724697276178, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.33998239040374756, "median": 0.3426361680030823, "p05": 0.2884981632232666, "p95": 0.38276687264442444 }, { "label": "completed_group_1", "mean": 0.2821658253669739, "median": 0.2775929868221283, "p05": 0.2536381781101227, "p95": 0.3205426335334778 }, { "label": "current_group_partial", "mean": 0.37785181403160095, "median": 0.3805788457393646, "p05": 0.34096643328666687, "p95": 0.40283167362213135 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.038641333580017, "group": 2, "latest_source_mass": 0.4193001687526703, "layer": 7, "max_source_mass": 0.4193001687526703, "mixer_index": 13, "normalized_entropy": 0.9454120842205417, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.34165239334106445, "median": 0.308063805103302, "p05": 0.2054935097694397, "p95": 0.6325749754905701 }, { "label": "completed_group_1", "mean": 0.23904743790626526, "median": 0.244267076253891, "p05": 0.1361692249774933, "p95": 0.30873122811317444 }, { "label": "current_group_partial", "mean": 0.4193001687526703, "median": 0.44425007700920105, "p05": 0.22311054170131683, "p95": 0.5117462277412415 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0754427909851074, "group": 2, "latest_source_mass": 0.3816372752189636, "layer": 8, "max_source_mass": 0.3816372752189636, "mixer_index": 14, "normalized_entropy": 0.9789102143476917, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3781457245349884, "median": 0.37959107756614685, "p05": 0.3339657485485077, "p95": 0.42053842544555664 }, { "label": "completed_group_1", "mean": 0.24021697044372559, "median": 0.23580573499202728, "p05": 0.21105526387691498, "p95": 0.2833511531352997 }, { "label": "current_group_partial", "mean": 0.3816372752189636, "median": 0.38216716051101685, "p05": 0.3468884229660034, "p95": 0.4153265357017517 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0433809757232666, "group": 2, "latest_source_mass": 0.4086577892303467, "layer": 8, "max_source_mass": 0.4086577892303467, "mixer_index": 15, "normalized_entropy": 0.9497262924195011, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.37802231311798096, "median": 0.3628900945186615, "p05": 0.27504366636276245, "p95": 0.5507957339286804 }, { "label": "completed_group_1", "mean": 0.21331989765167236, "median": 0.2002047300338745, "p05": 0.13782089948654175, "p95": 0.3198853135108948 }, { "label": "current_group_partial", "mean": 0.4086577892303467, "median": 0.41623425483703613, "p05": 0.29855865240097046, "p95": 0.47824543714523315 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0922105312347412, "group": 3, "latest_source_mass": 0.3464643955230713, "layer": 9, "max_source_mass": 0.3464643955230713, "mixer_index": 16, "normalized_entropy": 0.994172869264798, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3460429310798645, "median": 0.3446507453918457, "p05": 0.28670766949653625, "p95": 0.40976202487945557 }, { "label": "completed_group_1", "mean": 0.3074926435947418, "median": 0.306111603975296, "p05": 0.2859768867492676, "p95": 0.3337686061859131 }, { "label": "completed_group_2", "mean": 0.3464643955230713, "median": 0.3486866354942322, "p05": 0.27475792169570923, "p95": 0.40797942876815796 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3404325246810913, "group": 3, "latest_source_mass": 0.3108460605144501, "layer": 9, "max_source_mass": 0.3108460605144501, "mixer_index": 17, "normalized_entropy": 0.9669176780018418, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2880150079727173, "median": 0.27969539165496826, "p05": 0.18933802843093872, "p95": 0.45879441499710083 }, { "label": "completed_group_1", "mean": 0.16260020434856415, "median": 0.16053099930286407, "p05": 0.11509333550930023, "p95": 0.22219550609588623 }, { "label": "completed_group_2", "mean": 0.2385387420654297, "median": 0.24041923880577087, "p05": 0.1582493633031845, "p95": 0.2989943325519562 }, { "label": "current_group_partial", "mean": 0.3108460605144501, "median": 0.3127351701259613, "p05": 0.2580314874649048, "p95": 0.3545264005661011 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3765960931777954, "group": 3, "latest_source_mass": 0.24758204817771912, "layer": 10, "max_source_mass": 0.28703436255455017, "mixer_index": 18, "normalized_entropy": 0.9930041784673634, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28703436255455017, "median": 0.2873149514198303, "p05": 0.2421720325946808, "p95": 0.32964712381362915 }, { "label": "completed_group_1", "mean": 0.2552259564399719, "median": 0.2551601231098175, "p05": 0.2324453592300415, "p95": 0.27810582518577576 }, { "label": "completed_group_2", "mean": 0.2101576328277588, "median": 0.20999819040298462, "p05": 0.17491281032562256, "p95": 0.24711720645427704 }, { "label": "current_group_partial", "mean": 0.24758204817771912, "median": 0.2449823021888733, "p05": 0.21694661676883698, "p95": 0.2901700437068939 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3274900913238525, "group": 3, "latest_source_mass": 0.26038026809692383, "layer": 10, "max_source_mass": 0.33539247512817383, "mixer_index": 19, "normalized_entropy": 0.9575816857910796, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.33539247512817383, "median": 0.3373180031776428, "p05": 0.16021503508090973, "p95": 0.5567492246627808 }, { "label": "completed_group_1", "mean": 0.1987970769405365, "median": 0.19616791605949402, "p05": 0.13216234743595123, "p95": 0.2684130072593689 }, { "label": "completed_group_2", "mean": 0.20543015003204346, "median": 0.20896214246749878, "p05": 0.11321981251239777, "p95": 0.2748231887817383 }, { "label": "current_group_partial", "mean": 0.26038026809692383, "median": 0.2554967999458313, "p05": 0.1821153312921524, "p95": 0.3462901711463928 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3747133016586304, "group": 3, "latest_source_mass": 0.2833673655986786, "layer": 11, "max_source_mass": 0.2833673655986786, "mixer_index": 20, "normalized_entropy": 0.9916460314734998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.27988842129707336, "median": 0.28038889169692993, "p05": 0.24265244603157043, "p95": 0.31478387117385864 }, { "label": "completed_group_1", "mean": 0.2360810488462448, "median": 0.2335505485534668, "p05": 0.21358957886695862, "p95": 0.2655458450317383 }, { "label": "completed_group_2", "mean": 0.20066317915916443, "median": 0.2006566822528839, "p05": 0.18401089310646057, "p95": 0.21715889871120453 }, { "label": "current_group_partial", "mean": 0.2833673655986786, "median": 0.28357356786727905, "p05": 0.2585292458534241, "p95": 0.30662211775779724 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3239198923110962, "group": 3, "latest_source_mass": 0.2723969519138336, "layer": 11, "max_source_mass": 0.3361120820045471, "mixer_index": 21, "normalized_entropy": 0.9550063315857346, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3361120820045471, "median": 0.3591133654117584, "p05": 0.14645838737487793, "p95": 0.49824467301368713 }, { "label": "completed_group_1", "mean": 0.20296622812747955, "median": 0.19982439279556274, "p05": 0.13988786935806274, "p95": 0.27185869216918945 }, { "label": "completed_group_2", "mean": 0.1885247528553009, "median": 0.18602871894836426, "p05": 0.12637801468372345, "p95": 0.2539179027080536 }, { "label": "current_group_partial", "mean": 0.2723969519138336, "median": 0.25681596994400024, "p05": 0.19296783208847046, "p95": 0.3959115445613861 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3692121505737305, "group": 3, "latest_source_mass": 0.2974509000778198, "layer": 12, "max_source_mass": 0.2974509000778198, "mixer_index": 22, "normalized_entropy": 0.9876777897788168, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28167375922203064, "median": 0.27825769782066345, "p05": 0.23325958847999573, "p95": 0.3442809581756592 }, { "label": "completed_group_1", "mean": 0.21946553885936737, "median": 0.21849599480628967, "p05": 0.18336281180381775, "p95": 0.2548777461051941 }, { "label": "completed_group_2", "mean": 0.20140981674194336, "median": 0.2013496309518814, "p05": 0.1822294294834137, "p95": 0.2219773232936859 }, { "label": "current_group_partial", "mean": 0.2974509000778198, "median": 0.297912061214447, "p05": 0.26495662331581116, "p95": 0.32783159613609314 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3342361450195312, "group": 3, "latest_source_mass": 0.23993611335754395, "layer": 12, "max_source_mass": 0.36468517780303955, "mixer_index": 23, "normalized_entropy": 0.9624479348972428, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.36468517780303955, "median": 0.36720338463783264, "p05": 0.24960748851299286, "p95": 0.4882795810699463 }, { "label": "completed_group_1", "mean": 0.20593920350074768, "median": 0.2046196460723877, "p05": 0.16298335790634155, "p95": 0.25410282611846924 }, { "label": "completed_group_2", "mean": 0.18943950533866882, "median": 0.1881066858768463, "p05": 0.13994796574115753, "p95": 0.24192847311496735 }, { "label": "current_group_partial", "mean": 0.23993611335754395, "median": 0.232496976852417, "p05": 0.17571662366390228, "p95": 0.320015549659729 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3645737171173096, "group": 4, "latest_source_mass": 0.30504196882247925, "layer": 13, "max_source_mass": 0.30504196882247925, "mixer_index": 24, "normalized_entropy": 0.9843318673062809, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2597482204437256, "median": 0.2599886655807495, "p05": 0.23272822797298431, "p95": 0.2871353030204773 }, { "label": "completed_group_1", "mean": 0.22607669234275818, "median": 0.21320533752441406, "p05": 0.1705966293811798, "p95": 0.32829806208610535 }, { "label": "completed_group_2", "mean": 0.209133118391037, "median": 0.20912399888038635, "p05": 0.19090500473976135, "p95": 0.2265017032623291 }, { "label": "completed_group_3", "mean": 0.30504196882247925, "median": 0.3161492347717285, "p05": 0.20431193709373474, "p95": 0.37756454944610596 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5910131931304932, "group": 4, "latest_source_mass": 0.19691801071166992, "layer": 13, "max_source_mass": 0.2617476284503937, "mixer_index": 25, "normalized_entropy": 0.988552078237214, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2617476284503937, "median": 0.2677137851715088, "p05": 0.20086795091629028, "p95": 0.31516557931900024 }, { "label": "completed_group_1", "mean": 0.18299485743045807, "median": 0.1818893998861313, "p05": 0.1605435013771057, "p95": 0.20783139765262604 }, { "label": "completed_group_2", "mean": 0.17935845255851746, "median": 0.18049052357673645, "p05": 0.14385172724723816, "p95": 0.2110004425048828 }, { "label": "completed_group_3", "mean": 0.17898105084896088, "median": 0.17628446221351624, "p05": 0.14925207197666168, "p95": 0.21800483763217926 }, { "label": "current_group_partial", "mean": 0.19691801071166992, "median": 0.1964835375547409, "p05": 0.16946497559547424, "p95": 0.22626768052577972 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5884685516357422, "group": 4, "latest_source_mass": 0.26269519329071045, "layer": 14, "max_source_mass": 0.26269519329071045, "mixer_index": 26, "normalized_entropy": 0.9869710035805953, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2046908289194107, "median": 0.2052469551563263, "p05": 0.18031299114227295, "p95": 0.2295369803905487 }, { "label": "completed_group_1", "mean": 0.17294223606586456, "median": 0.16847725212574005, "p05": 0.15012723207473755, "p95": 0.21394622325897217 }, { "label": "completed_group_2", "mean": 0.15044090151786804, "median": 0.1493840515613556, "p05": 0.1370292603969574, "p95": 0.1660556048154831 }, { "label": "completed_group_3", "mean": 0.20923087000846863, "median": 0.20989257097244263, "p05": 0.17921577394008636, "p95": 0.23829039931297302 }, { "label": "current_group_partial", "mean": 0.26269519329071045, "median": 0.2656410038471222, "p05": 0.23202235996723175, "p95": 0.28119558095932007 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.599259853363037, "group": 4, "latest_source_mass": 0.21076934039592743, "layer": 14, "max_source_mass": 0.21657392382621765, "mixer_index": 27, "normalized_entropy": 0.9936760163331371, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21657392382621765, "median": 0.21635186672210693, "p05": 0.15601962804794312, "p95": 0.2927207350730896 }, { "label": "completed_group_1", "mean": 0.20580385625362396, "median": 0.20571252703666687, "p05": 0.18009206652641296, "p95": 0.2311524748802185 }, { "label": "completed_group_2", "mean": 0.19366896152496338, "median": 0.194598987698555, "p05": 0.16221287846565247, "p95": 0.22272710502147675 }, { "label": "completed_group_3", "mean": 0.1731838881969452, "median": 0.17174121737480164, "p05": 0.14892742037773132, "p95": 0.19922053813934326 }, { "label": "current_group_partial", "mean": 0.21076934039592743, "median": 0.20835596323013306, "p05": 0.1771993488073349, "p95": 0.2505151629447937 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5786192417144775, "group": 4, "latest_source_mass": 0.2778070569038391, "layer": 15, "max_source_mass": 0.2778070569038391, "mixer_index": 28, "normalized_entropy": 0.980851283245209, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20599736273288727, "median": 0.20488622784614563, "p05": 0.17115657031536102, "p95": 0.25322195887565613 }, { "label": "completed_group_1", "mean": 0.20593789219856262, "median": 0.20346379280090332, "p05": 0.18622605502605438, "p95": 0.2330799102783203 }, { "label": "completed_group_2", "mean": 0.13473063707351685, "median": 0.13439436256885529, "p05": 0.11930938065052032, "p95": 0.15121281147003174 }, { "label": "completed_group_3", "mean": 0.17552703619003296, "median": 0.1757315993309021, "p05": 0.1576612889766693, "p95": 0.1928577572107315 }, { "label": "current_group_partial", "mean": 0.2778070569038391, "median": 0.2790508270263672, "p05": 0.24338077008724213, "p95": 0.3088063597679138 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.580681324005127, "group": 4, "latest_source_mass": 0.2450503408908844, "layer": 15, "max_source_mass": 0.2450503408908844, "mixer_index": 29, "normalized_entropy": 0.9821325270103262, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21277281641960144, "median": 0.20640528202056885, "p05": 0.12015408277511597, "p95": 0.3659750819206238 }, { "label": "completed_group_1", "mean": 0.18522509932518005, "median": 0.1863747239112854, "p05": 0.1475280076265335, "p95": 0.21663622558116913 }, { "label": "completed_group_2", "mean": 0.18489867448806763, "median": 0.1888006329536438, "p05": 0.12606219947338104, "p95": 0.22459235787391663 }, { "label": "completed_group_3", "mean": 0.17205308377742767, "median": 0.17103317379951477, "p05": 0.13283535838127136, "p95": 0.20961183309555054 }, { "label": "current_group_partial", "mean": 0.2450503408908844, "median": 0.24511924386024475, "p05": 0.19535338878631592, "p95": 0.2939102053642273 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5705549716949463, "group": 4, "latest_source_mass": 0.28460633754730225, "layer": 16, "max_source_mass": 0.28460633754730225, "mixer_index": 30, "normalized_entropy": 0.9758406705603525, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2125052809715271, "median": 0.21049857139587402, "p05": 0.16735027730464935, "p95": 0.2640833854675293 }, { "label": "completed_group_1", "mean": 0.20224158465862274, "median": 0.20027190446853638, "p05": 0.16782984137535095, "p95": 0.24093782901763916 }, { "label": "completed_group_2", "mean": 0.12915468215942383, "median": 0.12795840203762054, "p05": 0.1122070699930191, "p95": 0.14902588725090027 }, { "label": "completed_group_3", "mean": 0.1714920997619629, "median": 0.1704360991716385, "p05": 0.15070456266403198, "p95": 0.19576270878314972 }, { "label": "current_group_partial", "mean": 0.28460633754730225, "median": 0.28339242935180664, "p05": 0.2418026626110077, "p95": 0.33516958355903625 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5599372386932373, "group": 4, "latest_source_mass": 0.2475198209285736, "layer": 16, "max_source_mass": 0.2665032148361206, "mixer_index": 31, "normalized_entropy": 0.9692435021205642, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2665032148361206, "median": 0.2523844242095947, "p05": 0.16442818939685822, "p95": 0.41760408878326416 }, { "label": "completed_group_1", "mean": 0.181569442152977, "median": 0.18072238564491272, "p05": 0.1432785987854004, "p95": 0.2251572459936142 }, { "label": "completed_group_2", "mean": 0.15844660997390747, "median": 0.15800738334655762, "p05": 0.11516115069389343, "p95": 0.20221972465515137 }, { "label": "completed_group_3", "mean": 0.14596089720726013, "median": 0.14446985721588135, "p05": 0.10661658644676208, "p95": 0.18743650615215302 }, { "label": "current_group_partial", "mean": 0.2475198209285736, "median": 0.24823713302612305, "p05": 0.1857534945011139, "p95": 0.30610787868499756 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.562828540802002, "group": 5, "latest_source_mass": 0.26428917050361633, "layer": 17, "max_source_mass": 0.26428917050361633, "mixer_index": 32, "normalized_entropy": 0.9710399691271056, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21806953847408295, "median": 0.21765722334384918, "p05": 0.1832873821258545, "p95": 0.256405234336853 }, { "label": "completed_group_1", "mean": 0.22324788570404053, "median": 0.2111721932888031, "p05": 0.1587892323732376, "p95": 0.33203551173210144 }, { "label": "completed_group_2", "mean": 0.11591021716594696, "median": 0.11326268315315247, "p05": 0.09832467138767242, "p95": 0.13705819845199585 }, { "label": "completed_group_3", "mean": 0.17848318815231323, "median": 0.1790149211883545, "p05": 0.1343652456998825, "p95": 0.21781212091445923 }, { "label": "completed_group_4", "mean": 0.26428917050361633, "median": 0.27161818742752075, "p05": 0.18240834772586823, "p95": 0.3144339919090271 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.761409044265747, "group": 5, "latest_source_mass": 0.14106416702270508, "layer": 17, "max_source_mass": 0.20728391408920288, "mixer_index": 33, "normalized_entropy": 0.9830611053081897, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1612042486667633, "median": 0.14052782952785492, "p05": 0.09785228967666626, "p95": 0.30037230253219604 }, { "label": "completed_group_1", "mean": 0.16519047319889069, "median": 0.16112321615219116, "p05": 0.13238948583602905, "p95": 0.2119981199502945 }, { "label": "completed_group_2", "mean": 0.17529632151126862, "median": 0.18006199598312378, "p05": 0.12263078987598419, "p95": 0.20985063910484314 }, { "label": "completed_group_3", "mean": 0.14996090531349182, "median": 0.1468663513660431, "p05": 0.11909312009811401, "p95": 0.18974041938781738 }, { "label": "completed_group_4", "mean": 0.20728391408920288, "median": 0.2042727917432785, "p05": 0.1449805498123169, "p95": 0.27828457951545715 }, { "label": "current_group_partial", "mean": 0.14106416702270508, "median": 0.1405012309551239, "p05": 0.10205969214439392, "p95": 0.1835198849439621 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7366840839385986, "group": 5, "latest_source_mass": 0.2530929446220398, "layer": 18, "max_source_mass": 0.2530929446220398, "mixer_index": 34, "normalized_entropy": 0.9692618422085502, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.18514017760753632, "median": 0.18704235553741455, "p05": 0.13617590069770813, "p95": 0.22916007041931152 }, { "label": "completed_group_1", "mean": 0.1663871705532074, "median": 0.16571956872940063, "p05": 0.1462152749300003, "p95": 0.18810859322547913 }, { "label": "completed_group_2", "mean": 0.09063515067100525, "median": 0.08811916410923004, "p05": 0.07649004459381104, "p95": 0.11148498952388763 }, { "label": "completed_group_3", "mean": 0.11971023678779602, "median": 0.11747527122497559, "p05": 0.10390099883079529, "p95": 0.1423199474811554 }, { "label": "completed_group_4", "mean": 0.18503433465957642, "median": 0.18477493524551392, "p05": 0.15245835483074188, "p95": 0.22021599113941193 }, { "label": "current_group_partial", "mean": 0.2530929446220398, "median": 0.25188589096069336, "p05": 0.22554033994674683, "p95": 0.2833186984062195 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.777672290802002, "group": 5, "latest_source_mass": 0.19777557253837585, "layer": 18, "max_source_mass": 0.19777557253837585, "mixer_index": 35, "normalized_entropy": 0.9921377960222963, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.15483248233795166, "median": 0.14712893962860107, "p05": 0.12276758253574371, "p95": 0.22418904304504395 }, { "label": "completed_group_1", "mean": 0.16576308012008667, "median": 0.16587528586387634, "p05": 0.13462601602077484, "p95": 0.19553470611572266 }, { "label": "completed_group_2", "mean": 0.16957050561904907, "median": 0.17099428176879883, "p05": 0.14377880096435547, "p95": 0.19032810628414154 }, { "label": "completed_group_3", "mean": 0.14225168526172638, "median": 0.14081379771232605, "p05": 0.12333962321281433, "p95": 0.1642245650291443 }, { "label": "completed_group_4", "mean": 0.16980668902397156, "median": 0.16821962594985962, "p05": 0.13770604133605957, "p95": 0.20901428163051605 }, { "label": "current_group_partial", "mean": 0.19777557253837585, "median": 0.19705063104629517, "p05": 0.1461065709590912, "p95": 0.2508929371833801 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7414222955703735, "group": 5, "latest_source_mass": 0.25494134426116943, "layer": 19, "max_source_mass": 0.25494134426116943, "mixer_index": 36, "normalized_entropy": 0.9719062884710925, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18020270764827728, "median": 0.180809885263443, "p05": 0.1366642266511917, "p95": 0.22788876295089722 }, { "label": "completed_group_1", "mean": 0.1900298148393631, "median": 0.1896573156118393, "p05": 0.17211011052131653, "p95": 0.20767614245414734 }, { "label": "completed_group_2", "mean": 0.09944289177656174, "median": 0.09800659120082855, "p05": 0.08568179607391357, "p95": 0.11607980728149414 }, { "label": "completed_group_3", "mean": 0.11960913985967636, "median": 0.11915834993124008, "p05": 0.10471149533987045, "p95": 0.1360406130552292 }, { "label": "completed_group_4", "mean": 0.1557741016149521, "median": 0.15596863627433777, "p05": 0.13432671129703522, "p95": 0.1770717203617096 }, { "label": "current_group_partial", "mean": 0.25494134426116943, "median": 0.25551509857177734, "p05": 0.22669723629951477, "p95": 0.2819991707801819 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.733978509902954, "group": 5, "latest_source_mass": 0.2618475556373596, "layer": 19, "max_source_mass": 0.2618475556373596, "mixer_index": 37, "normalized_entropy": 0.9677518325883359, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15204599499702454, "median": 0.1308840811252594, "p05": 0.09369519352912903, "p95": 0.32571133971214294 }, { "label": "completed_group_1", "mean": 0.14166373014450073, "median": 0.14088967442512512, "p05": 0.11298352479934692, "p95": 0.17192427814006805 }, { "label": "completed_group_2", "mean": 0.1616208553314209, "median": 0.1641538441181183, "p05": 0.11296901851892471, "p95": 0.1948811560869217 }, { "label": "completed_group_3", "mean": 0.1256796270608902, "median": 0.122653529047966, "p05": 0.09500046819448471, "p95": 0.16056732833385468 }, { "label": "completed_group_4", "mean": 0.1571422517299652, "median": 0.1528569459915161, "p05": 0.11623523384332657, "p95": 0.21299627423286438 }, { "label": "current_group_partial", "mean": 0.2618475556373596, "median": 0.26793089509010315, "p05": 0.15509119629859924, "p95": 0.339982271194458 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7344707250595093, "group": 5, "latest_source_mass": 0.2615981698036194, "layer": 20, "max_source_mass": 0.2615981698036194, "mixer_index": 38, "normalized_entropy": 0.9680265430977588, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1829337775707245, "median": 0.183739572763443, "p05": 0.13839715719223022, "p95": 0.2284226417541504 }, { "label": "completed_group_1", "mean": 0.18416839838027954, "median": 0.18364182114601135, "p05": 0.16665373742580414, "p95": 0.20437687635421753 }, { "label": "completed_group_2", "mean": 0.09014187008142471, "median": 0.08851997554302216, "p05": 0.07715612649917603, "p95": 0.10642829537391663 }, { "label": "completed_group_3", "mean": 0.12335794419050217, "median": 0.12144016474485397, "p05": 0.10556001961231232, "p95": 0.14631223678588867 }, { "label": "completed_group_4", "mean": 0.15779981017112732, "median": 0.15796968340873718, "p05": 0.1340107023715973, "p95": 0.18188069760799408 }, { "label": "current_group_partial", "mean": 0.2615981698036194, "median": 0.262384295463562, "p05": 0.230574831366539, "p95": 0.28982529044151306 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.735896348953247, "group": 5, "latest_source_mass": 0.27010154724121094, "layer": 20, "max_source_mass": 0.27010154724121094, "mixer_index": 39, "normalized_entropy": 0.9688221989423192, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.17276549339294434, "median": 0.16886113584041595, "p05": 0.10281073302030563, "p95": 0.2923668622970581 }, { "label": "completed_group_1", "mean": 0.15101772546768188, "median": 0.14904022216796875, "p05": 0.13034172356128693, "p95": 0.17705516517162323 }, { "label": "completed_group_2", "mean": 0.13220664858818054, "median": 0.1315174400806427, "p05": 0.10168404132127762, "p95": 0.16267849504947662 }, { "label": "completed_group_3", "mean": 0.11932596564292908, "median": 0.11731794476509094, "p05": 0.09613680094480515, "p95": 0.1465223878622055 }, { "label": "completed_group_4", "mean": 0.15458261966705322, "median": 0.15350666642189026, "p05": 0.12296099960803986, "p95": 0.19313591718673706 }, { "label": "current_group_partial", "mean": 0.27010154724121094, "median": 0.27393096685409546, "p05": 0.19484001398086548, "p95": 0.3264923095703125 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7644665241241455, "group": 6, "latest_source_mass": 0.2094498574733734, "layer": 21, "max_source_mass": 0.2094498574733734, "mixer_index": 40, "normalized_entropy": 0.9847675173076283, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.19160009920597076, "median": 0.196849524974823, "p05": 0.12982682883739471, "p95": 0.24405300617218018 }, { "label": "completed_group_1", "mean": 0.1915181279182434, "median": 0.1907939314842224, "p05": 0.17317205667495728, "p95": 0.2111247181892395 }, { "label": "completed_group_2", "mean": 0.11520625650882721, "median": 0.11408726871013641, "p05": 0.10055835545063019, "p95": 0.13332504034042358 }, { "label": "completed_group_3", "mean": 0.1341419219970703, "median": 0.13221079111099243, "p05": 0.11641807854175568, "p95": 0.15773889422416687 }, { "label": "completed_group_4", "mean": 0.1580837219953537, "median": 0.15754032135009766, "p05": 0.13662610948085785, "p95": 0.18392926454544067 }, { "label": "completed_group_5", "mean": 0.2094498574733734, "median": 0.20958366990089417, "p05": 0.1805649697780609, "p95": 0.23726987838745117 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6835086345672607, "group": 6, "latest_source_mass": 0.43279585242271423, "layer": 21, "max_source_mass": 0.43279585242271423, "mixer_index": 41, "normalized_entropy": 0.8651522966692777, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10065847635269165, "median": 0.09005473554134369, "p05": 0.05753578990697861, "p95": 0.19214311242103577 }, { "label": "completed_group_1", "mean": 0.09339895844459534, "median": 0.09201203286647797, "p05": 0.07428346574306488, "p95": 0.11678388714790344 }, { "label": "completed_group_2", "mean": 0.09682720899581909, "median": 0.09625889360904694, "p05": 0.07657647132873535, "p95": 0.11835116147994995 }, { "label": "completed_group_3", "mean": 0.08155890554189682, "median": 0.08032877743244171, "p05": 0.06701995432376862, "p95": 0.09893998503684998 }, { "label": "completed_group_4", "mean": 0.09126688539981842, "median": 0.09069330245256424, "p05": 0.07231517881155014, "p95": 0.1128317192196846 }, { "label": "completed_group_5", "mean": 0.10349369794130325, "median": 0.10402766615152359, "p05": 0.07336798310279846, "p95": 0.1318768411874771 }, { "label": "current_group_partial", "mean": 0.43279585242271423, "median": 0.4367910623550415, "p05": 0.3485929071903229, "p95": 0.5040479302406311 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7937740087509155, "group": 6, "latest_source_mass": 0.3452628254890442, "layer": 22, "max_source_mass": 0.3452628254890442, "mixer_index": 42, "normalized_entropy": 0.9218174896830382, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13723835349082947, "median": 0.1412133425474167, "p05": 0.08910952508449554, "p95": 0.17562061548233032 }, { "label": "completed_group_1", "mean": 0.12937891483306885, "median": 0.13051339983940125, "p05": 0.10541269928216934, "p95": 0.1487061083316803 }, { "label": "completed_group_2", "mean": 0.07450413703918457, "median": 0.07415075600147247, "p05": 0.06578464806079865, "p95": 0.08426731824874878 }, { "label": "completed_group_3", "mean": 0.08537043631076813, "median": 0.08443015813827515, "p05": 0.07344960421323776, "p95": 0.09956531971693039 }, { "label": "completed_group_4", "mean": 0.09880410879850388, "median": 0.09825065732002258, "p05": 0.08438277244567871, "p95": 0.11530908197164536 }, { "label": "completed_group_5", "mean": 0.12944120168685913, "median": 0.12968069314956665, "p05": 0.10858328640460968, "p95": 0.15059794485569 }, { "label": "current_group_partial", "mean": 0.3452628254890442, "median": 0.33866745233535767, "p05": 0.288330078125, "p95": 0.42612025141716003 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5790722370147705, "group": 6, "latest_source_mass": 0.49195876717567444, "layer": 22, "max_source_mass": 0.49195876717567444, "mixer_index": 43, "normalized_entropy": 0.8114826050839847, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09500400722026825, "median": 0.07923799753189087, "p05": 0.045484717935323715, "p95": 0.2259966880083084 }, { "label": "completed_group_1", "mean": 0.08892714232206345, "median": 0.0876917839050293, "p05": 0.06900187581777573, "p95": 0.11303731799125671 }, { "label": "completed_group_2", "mean": 0.09081116318702698, "median": 0.0898551493883133, "p05": 0.06699725985527039, "p95": 0.11749734729528427 }, { "label": "completed_group_3", "mean": 0.07538282871246338, "median": 0.07381957024335861, "p05": 0.05869410187005997, "p95": 0.09544046223163605 }, { "label": "completed_group_4", "mean": 0.07685283571481705, "median": 0.07580418884754181, "p05": 0.056799523532390594, "p95": 0.10076010227203369 }, { "label": "completed_group_5", "mean": 0.08106326311826706, "median": 0.08067160844802856, "p05": 0.05696551501750946, "p95": 0.10726240277290344 }, { "label": "current_group_partial", "mean": 0.49195876717567444, "median": 0.49396610260009766, "p05": 0.3897285759449005, "p95": 0.583945631980896 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8034920692443848, "group": 6, "latest_source_mass": 0.33595213294029236, "layer": 23, "max_source_mass": 0.33595213294029236, "mixer_index": 44, "normalized_entropy": 0.926811584861681, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.142653226852417, "median": 0.14694267511367798, "p05": 0.08785572648048401, "p95": 0.18721188604831696 }, { "label": "completed_group_1", "mean": 0.13419847190380096, "median": 0.13438312709331512, "p05": 0.11451410502195358, "p95": 0.15392597019672394 }, { "label": "completed_group_2", "mean": 0.07437049597501755, "median": 0.07391499727964401, "p05": 0.06574448198080063, "p95": 0.08430380374193192 }, { "label": "completed_group_3", "mean": 0.09070361405611038, "median": 0.08887749910354614, "p05": 0.07508289068937302, "p95": 0.11117327958345413 }, { "label": "completed_group_4", "mean": 0.1013825386762619, "median": 0.10140319168567657, "p05": 0.08447366952896118, "p95": 0.11889617145061493 }, { "label": "completed_group_5", "mean": 0.12073953449726105, "median": 0.12033741921186447, "p05": 0.10022275149822235, "p95": 0.1415012627840042 }, { "label": "current_group_partial", "mean": 0.33595213294029236, "median": 0.330989271402359, "p05": 0.27680477499961853, "p95": 0.41842275857925415 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5278494358062744, "group": 6, "latest_source_mass": 0.51820969581604, "layer": 23, "max_source_mass": 0.51820969581604, "mixer_index": 45, "normalized_entropy": 0.7851592924514033, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10045768320560455, "median": 0.08486579358577728, "p05": 0.04896760731935501, "p95": 0.2206544280052185 }, { "label": "completed_group_1", "mean": 0.08844950795173645, "median": 0.08638796210289001, "p05": 0.06730383634567261, "p95": 0.11565113067626953 }, { "label": "completed_group_2", "mean": 0.08561251312494278, "median": 0.08505219221115112, "p05": 0.0625130906701088, "p95": 0.11068116128444672 }, { "label": "completed_group_3", "mean": 0.06598107516765594, "median": 0.06433478742837906, "p05": 0.04859655350446701, "p95": 0.08642670512199402 }, { "label": "completed_group_4", "mean": 0.06767018139362335, "median": 0.06678327918052673, "p05": 0.047009289264678955, "p95": 0.09129223972558975 }, { "label": "completed_group_5", "mean": 0.07361933588981628, "median": 0.07290077209472656, "p05": 0.049975596368312836, "p95": 0.09868258237838745 }, { "label": "current_group_partial", "mean": 0.51820969581604, "median": 0.5205609798431396, "p05": 0.40315571427345276, "p95": 0.6213710308074951 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8435695171356201, "group": 6, "latest_source_mass": 0.29524505138397217, "layer": 24, "max_source_mass": 0.29524505138397217, "mixer_index": 46, "normalized_entropy": 0.9474073188993969, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14839638769626617, "median": 0.15232303738594055, "p05": 0.08978340029716492, "p95": 0.19894656538963318 }, { "label": "completed_group_1", "mean": 0.1482572853565216, "median": 0.14857065677642822, "p05": 0.13154153525829315, "p95": 0.16430144011974335 }, { "label": "completed_group_2", "mean": 0.08113027364015579, "median": 0.08067925274372101, "p05": 0.07102686911821365, "p95": 0.09267604351043701 }, { "label": "completed_group_3", "mean": 0.09534022957086563, "median": 0.09320184588432312, "p05": 0.0778813511133194, "p95": 0.11805401742458344 }, { "label": "completed_group_4", "mean": 0.10612697899341583, "median": 0.10644470155239105, "p05": 0.08815888315439224, "p95": 0.12415381520986557 }, { "label": "completed_group_5", "mean": 0.12550383806228638, "median": 0.1254296898841858, "p05": 0.10509618371725082, "p95": 0.14616522192955017 }, { "label": "current_group_partial", "mean": 0.29524505138397217, "median": 0.29100558161735535, "p05": 0.24107839167118073, "p95": 0.37045177817344666 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6421003341674805, "group": 6, "latest_source_mass": 0.4530346691608429, "layer": 24, "max_source_mass": 0.4530346691608429, "mixer_index": 47, "normalized_entropy": 0.8438726397334819, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11422763019800186, "median": 0.09945468604564667, "p05": 0.06323201954364777, "p95": 0.23352834582328796 }, { "label": "completed_group_1", "mean": 0.10543527454137802, "median": 0.1044958084821701, "p05": 0.08478721231222153, "p95": 0.1284603774547577 }, { "label": "completed_group_2", "mean": 0.09285669028759003, "median": 0.09177995473146439, "p05": 0.06902404874563217, "p95": 0.12035554647445679 }, { "label": "completed_group_3", "mean": 0.07746599614620209, "median": 0.07653152942657471, "p05": 0.058040909469127655, "p95": 0.09812526404857635 }, { "label": "completed_group_4", "mean": 0.07571054995059967, "median": 0.07457360625267029, "p05": 0.05648855119943619, "p95": 0.0989091694355011 }, { "label": "completed_group_5", "mean": 0.08126918971538544, "median": 0.08049014210700989, "p05": 0.05931335315108299, "p95": 0.10596252232789993 }, { "label": "current_group_partial", "mean": 0.4530346691608429, "median": 0.455755352973938, "p05": 0.3413957357406616, "p95": 0.5519019365310669 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8733552694320679, "group": 7, "latest_source_mass": 0.26451510190963745, "layer": 25, "max_source_mass": 0.26451510190963745, "mixer_index": 48, "normalized_entropy": 0.9627141676307774, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14798463881015778, "median": 0.14657661318778992, "p05": 0.09472498297691345, "p95": 0.2258487045764923 }, { "label": "completed_group_1", "mean": 0.160526841878891, "median": 0.16084568202495575, "p05": 0.1358838975429535, "p95": 0.18423213064670563 }, { "label": "completed_group_2", "mean": 0.09668850898742676, "median": 0.09659601747989655, "p05": 0.08555105328559875, "p95": 0.1084243580698967 }, { "label": "completed_group_3", "mean": 0.10477185994386673, "median": 0.10316876322031021, "p05": 0.08876212686300278, "p95": 0.12562818825244904 }, { "label": "completed_group_4", "mean": 0.10857057571411133, "median": 0.1074000895023346, "p05": 0.09068692475557327, "p95": 0.12911471724510193 }, { "label": "completed_group_5", "mean": 0.11694245785474777, "median": 0.11679476499557495, "p05": 0.0976894274353981, "p95": 0.13528725504875183 }, { "label": "completed_group_6", "mean": 0.26451510190963745, "median": 0.26442092657089233, "p05": 0.21035388112068176, "p95": 0.31906524300575256 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.784506916999817, "group": 7, "latest_source_mass": 0.27427971363067627, "layer": 25, "max_source_mass": 0.29875272512435913, "mixer_index": 49, "normalized_entropy": 0.8581664265292297, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08589775860309601, "median": 0.07514797151088715, "p05": 0.03144541755318642, "p95": 0.20497030019760132 }, { "label": "completed_group_1", "mean": 0.08786046504974365, "median": 0.08646859228610992, "p05": 0.06123892217874527, "p95": 0.11937689781188965 }, { "label": "completed_group_2", "mean": 0.07714234292507172, "median": 0.07644426077604294, "p05": 0.05296650156378746, "p95": 0.10448302328586578 }, { "label": "completed_group_3", "mean": 0.06314892321825027, "median": 0.06132982298731804, "p05": 0.044174306094646454, "p95": 0.08466912060976028 }, { "label": "completed_group_4", "mean": 0.05684571713209152, "median": 0.055630408227443695, "p05": 0.036537639796733856, "p95": 0.08176163583993912 }, { "label": "completed_group_5", "mean": 0.056072384119033813, "median": 0.0549161471426487, "p05": 0.03825453296303749, "p95": 0.07769857347011566 }, { "label": "completed_group_6", "mean": 0.29875272512435913, "median": 0.29761478304862976, "p05": 0.17369259893894196, "p95": 0.4214131534099579 }, { "label": "current_group_partial", "mean": 0.27427971363067627, "median": 0.26517564058303833, "p05": 0.11691330373287201, "p95": 0.45670589804649353 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9675533771514893, "group": 7, "latest_source_mass": 0.24859297275543213, "layer": 26, "max_source_mass": 0.24859297275543213, "mixer_index": 50, "normalized_entropy": 0.9461931666335953, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11549945175647736, "median": 0.1186310350894928, "p05": 0.0548042394220829, "p95": 0.17459861934185028 }, { "label": "completed_group_1", "mean": 0.12592050433158875, "median": 0.12912830710411072, "p05": 0.08620737493038177, "p95": 0.15710318088531494 }, { "label": "completed_group_2", "mean": 0.07530814409255981, "median": 0.07495854794979095, "p05": 0.06174474209547043, "p95": 0.0894969180226326 }, { "label": "completed_group_3", "mean": 0.08277833461761475, "median": 0.08158879727125168, "p05": 0.06734651327133179, "p95": 0.10064717382192612 }, { "label": "completed_group_4", "mean": 0.08296641707420349, "median": 0.08213280141353607, "p05": 0.06705781072378159, "p95": 0.10095006227493286 }, { "label": "completed_group_5", "mean": 0.09016789495944977, "median": 0.08980530500411987, "p05": 0.07541801035404205, "p95": 0.10646115988492966 }, { "label": "completed_group_6", "mean": 0.17876628041267395, "median": 0.17772917449474335, "p05": 0.13528534770011902, "p95": 0.2264900654554367 }, { "label": "current_group_partial", "mean": 0.24859297275543213, "median": 0.245039701461792, "p05": 0.14620795845985413, "p95": 0.3620130121707916 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7119876146316528, "group": 7, "latest_source_mass": 0.3658455014228821, "layer": 26, "max_source_mass": 0.3658455014228821, "mixer_index": 51, "normalized_entropy": 0.8232920138974705, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08804735541343689, "median": 0.08122549951076508, "p05": 0.054039862006902695, "p95": 0.1443495899438858 }, { "label": "completed_group_1", "mean": 0.07527871429920197, "median": 0.0738755539059639, "p05": 0.05371527001261711, "p95": 0.10149700939655304 }, { "label": "completed_group_2", "mean": 0.05666808784008026, "median": 0.05508599430322647, "p05": 0.03937675803899765, "p95": 0.07947445660829544 }, { "label": "completed_group_3", "mean": 0.050419021397829056, "median": 0.04877372086048126, "p05": 0.03257015347480774, "p95": 0.0718536451458931 }, { "label": "completed_group_4", "mean": 0.05142981931567192, "median": 0.0498797781765461, "p05": 0.034862153232097626, "p95": 0.07353642582893372 }, { "label": "completed_group_5", "mean": 0.05313318967819214, "median": 0.051505643874406815, "p05": 0.037266045808792114, "p95": 0.07490143924951553 }, { "label": "completed_group_6", "mean": 0.2591783106327057, "median": 0.2531449794769287, "p05": 0.15121260285377502, "p95": 0.386227548122406 }, { "label": "current_group_partial", "mean": 0.3658455014228821, "median": 0.3656759262084961, "p05": 0.19218485057353973, "p95": 0.5332037210464478 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9094492197036743, "group": 7, "latest_source_mass": 0.2766643464565277, "layer": 27, "max_source_mass": 0.2766643464565277, "mixer_index": 52, "normalized_entropy": 0.918250973365264, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10337945818901062, "median": 0.10516610741615295, "p05": 0.05207033082842827, "p95": 0.1573200225830078 }, { "label": "completed_group_1", "mean": 0.10583682358264923, "median": 0.10595200955867767, "p05": 0.0643514022231102, "p95": 0.14548130333423615 }, { "label": "completed_group_2", "mean": 0.08642244338989258, "median": 0.08549235016107559, "p05": 0.06607823818922043, "p95": 0.11075147986412048 }, { "label": "completed_group_3", "mean": 0.08043928444385529, "median": 0.07886679470539093, "p05": 0.05936145409941673, "p95": 0.10334464907646179 }, { "label": "completed_group_4", "mean": 0.0717030018568039, "median": 0.07051880657672882, "p05": 0.05472247302532196, "p95": 0.09299077093601227 }, { "label": "completed_group_5", "mean": 0.0645938515663147, "median": 0.06373226642608643, "p05": 0.04421529173851013, "p95": 0.08712958544492722 }, { "label": "completed_group_6", "mean": 0.21096080541610718, "median": 0.20807480812072754, "p05": 0.14586198329925537, "p95": 0.2868158519268036 }, { "label": "current_group_partial", "mean": 0.2766643464565277, "median": 0.27722808718681335, "p05": 0.16094130277633667, "p95": 0.3963583707809448 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7860023975372314, "group": 7, "latest_source_mass": 0.3244447112083435, "layer": 27, "max_source_mass": 0.3244447112083435, "mixer_index": 53, "normalized_entropy": 0.8588856006475877, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09247160702943802, "median": 0.08733023703098297, "p05": 0.04943867400288582, "p95": 0.16303153336048126 }, { "label": "completed_group_1", "mean": 0.0784597396850586, "median": 0.07711629569530487, "p05": 0.05584436655044556, "p95": 0.10506026446819305 }, { "label": "completed_group_2", "mean": 0.07236532866954803, "median": 0.07097338140010834, "p05": 0.050456635653972626, "p95": 0.09917968511581421 }, { "label": "completed_group_3", "mean": 0.05986309051513672, "median": 0.057845089584589005, "p05": 0.04075033217668533, "p95": 0.08416998386383057 }, { "label": "completed_group_4", "mean": 0.05389959365129471, "median": 0.05301224812865257, "p05": 0.03848717734217644, "p95": 0.07268592715263367 }, { "label": "completed_group_5", "mean": 0.06059400737285614, "median": 0.05963316559791565, "p05": 0.04356392100453377, "p95": 0.08111730217933655 }, { "label": "completed_group_6", "mean": 0.2579019069671631, "median": 0.2530352473258972, "p05": 0.16421708464622498, "p95": 0.3691956102848053 }, { "label": "current_group_partial", "mean": 0.3244447112083435, "median": 0.3252047896385193, "p05": 0.17263495922088623, "p95": 0.47815465927124023 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9232951402664185, "group": 7, "latest_source_mass": 0.2509572505950928, "layer": 28, "max_source_mass": 0.2509572505950928, "mixer_index": 54, "normalized_entropy": 0.9249094536760685, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1160542219877243, "median": 0.1222282201051712, "p05": 0.061863381415605545, "p95": 0.16598135232925415 }, { "label": "completed_group_1", "mean": 0.11346358060836792, "median": 0.11177559196949005, "p05": 0.05926967412233353, "p95": 0.16970883309841156 }, { "label": "completed_group_2", "mean": 0.08816123753786087, "median": 0.08707751333713531, "p05": 0.06823235750198364, "p95": 0.1114775538444519 }, { "label": "completed_group_3", "mean": 0.08138670027256012, "median": 0.0797908753156662, "p05": 0.06149819865822792, "p95": 0.10523255914449692 }, { "label": "completed_group_4", "mean": 0.07161372900009155, "median": 0.07095701992511749, "p05": 0.055086471140384674, "p95": 0.09069444239139557 }, { "label": "completed_group_5", "mean": 0.06013647839426994, "median": 0.05960702896118164, "p05": 0.04596705362200737, "p95": 0.076435886323452 }, { "label": "completed_group_6", "mean": 0.21822679042816162, "median": 0.21632584929466248, "p05": 0.14715570211410522, "p95": 0.2940847873687744 }, { "label": "current_group_partial", "mean": 0.2509572505950928, "median": 0.24700899422168732, "p05": 0.14031530916690826, "p95": 0.37019768357276917 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8294003009796143, "group": 7, "latest_source_mass": 0.20826560258865356, "layer": 28, "max_source_mass": 0.31431835889816284, "mixer_index": 55, "normalized_entropy": 0.879755580674689, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08831655979156494, "median": 0.06431648135185242, "p05": 0.03252303972840309, "p95": 0.2706584930419922 }, { "label": "completed_group_1", "mean": 0.10072161257266998, "median": 0.1009141132235527, "p05": 0.06054181233048439, "p95": 0.14067479968070984 }, { "label": "completed_group_2", "mean": 0.0818098932504654, "median": 0.08032949268817902, "p05": 0.054586175829172134, "p95": 0.1145491972565651 }, { "label": "completed_group_3", "mean": 0.07514279335737228, "median": 0.07337591052055359, "p05": 0.04998338222503662, "p95": 0.10431315004825592 }, { "label": "completed_group_4", "mean": 0.06651485711336136, "median": 0.06524930149316788, "p05": 0.04414192587137222, "p95": 0.09315875917673111 }, { "label": "completed_group_5", "mean": 0.06491032987833023, "median": 0.06368150562047958, "p05": 0.04332352057099342, "p95": 0.09065432846546173 }, { "label": "completed_group_6", "mean": 0.31431835889816284, "median": 0.3147735297679901, "p05": 0.18699732422828674, "p95": 0.44058555364608765 }, { "label": "current_group_partial", "mean": 0.20826560258865356, "median": 0.1914007067680359, "p05": 0.08225662261247635, "p95": 0.38793477416038513 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9393219947814941, "group": 8, "latest_source_mass": 0.15344023704528809, "layer": 29, "max_source_mass": 0.2604207992553711, "mixer_index": 56, "normalized_entropy": 0.9326167415193847, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11041727662086487, "median": 0.10045937448740005, "p05": 0.024797730147838593, "p95": 0.279465913772583 }, { "label": "completed_group_1", "mean": 0.12080886960029602, "median": 0.12561748921871185, "p05": 0.0680447518825531, "p95": 0.16192926466464996 }, { "label": "completed_group_2", "mean": 0.09995828568935394, "median": 0.09907965362071991, "p05": 0.07490158826112747, "p95": 0.1276247799396515 }, { "label": "completed_group_3", "mean": 0.09934850037097931, "median": 0.09762611985206604, "p05": 0.06941570341587067, "p95": 0.134530171751976 }, { "label": "completed_group_4", "mean": 0.08031868189573288, "median": 0.07758662104606628, "p05": 0.05694210156798363, "p95": 0.11248748749494553 }, { "label": "completed_group_5", "mean": 0.075287364423275, "median": 0.07397153228521347, "p05": 0.05265223979949951, "p95": 0.10342685133218765 }, { "label": "completed_group_6", "mean": 0.2604207992553711, "median": 0.2616186738014221, "p05": 0.16666486859321594, "p95": 0.34887272119522095 }, { "label": "completed_group_7", "mean": 0.15344023704528809, "median": 0.13995957374572754, "p05": 0.05727577582001686, "p95": 0.29449376463890076 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.9925593137741089, "group": 8, "latest_source_mass": 0.18462148308753967, "layer": 29, "max_source_mass": 0.2503892779350281, "mixer_index": 57, "normalized_entropy": 0.9068528243889232, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08698172122240067, "median": 0.07842238247394562, "p05": 0.02795225754380226, "p95": 0.1876923143863678 }, { "label": "completed_group_1", "mean": 0.08857265114784241, "median": 0.08714614808559418, "p05": 0.06365615874528885, "p95": 0.11900493502616882 }, { "label": "completed_group_2", "mean": 0.07299254834651947, "median": 0.07189398258924484, "p05": 0.05586059391498566, "p95": 0.0933830514550209 }, { "label": "completed_group_3", "mean": 0.060478877276182175, "median": 0.05862775444984436, "p05": 0.04474102333188057, "p95": 0.07903359085321426 }, { "label": "completed_group_4", "mean": 0.054115522652864456, "median": 0.052696481347084045, "p05": 0.03536852449178696, "p95": 0.07748772948980331 }, { "label": "completed_group_5", "mean": 0.056821923702955246, "median": 0.05573669448494911, "p05": 0.0398341529071331, "p95": 0.07687154412269592 }, { "label": "completed_group_6", "mean": 0.2503892779350281, "median": 0.2475910186767578, "p05": 0.15396791696548462, "p95": 0.357724130153656 }, { "label": "completed_group_7", "mean": 0.14502598345279694, "median": 0.13307344913482666, "p05": 0.06183449178934097, "p95": 0.2673647403717041 }, { "label": "current_group_partial", "mean": 0.18462148308753967, "median": 0.17473986744880676, "p05": 0.08267536759376526, "p95": 0.3240666389465332 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.068819999694824, "group": 8, "latest_source_mass": 0.15494346618652344, "layer": 30, "max_source_mass": 0.21415996551513672, "mixer_index": 58, "normalized_entropy": 0.9415605582761754, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09429103136062622, "median": 0.0943363606929779, "p05": 0.03544093295931816, "p95": 0.16596338152885437 }, { "label": "completed_group_1", "mean": 0.09028280526399612, "median": 0.090491384267807, "p05": 0.05409309267997742, "p95": 0.12526515126228333 }, { "label": "completed_group_2", "mean": 0.09188266098499298, "median": 0.09010649472475052, "p05": 0.06878700107336044, "p95": 0.12181076407432556 }, { "label": "completed_group_3", "mean": 0.0735517293214798, "median": 0.07178769260644913, "p05": 0.0548262856900692, "p95": 0.09550250321626663 }, { "label": "completed_group_4", "mean": 0.06746294349431992, "median": 0.06608930230140686, "p05": 0.050633110105991364, "p95": 0.08874846994876862 }, { "label": "completed_group_5", "mean": 0.060166649520397186, "median": 0.060052454471588135, "p05": 0.039401471614837646, "p95": 0.08220525085926056 }, { "label": "completed_group_6", "mean": 0.21415996551513672, "median": 0.2118481993675232, "p05": 0.15205052495002747, "p95": 0.2836942672729492 }, { "label": "completed_group_7", "mean": 0.15325874090194702, "median": 0.14970678091049194, "p05": 0.08526452630758286, "p95": 0.23864519596099854 }, { "label": "current_group_partial", "mean": 0.15494346618652344, "median": 0.1473558098077774, "p05": 0.0697651207447052, "p95": 0.2689872086048126 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0139355659484863, "group": 8, "latest_source_mass": 0.1372738927602768, "layer": 30, "max_source_mass": 0.24272148311138153, "mixer_index": 59, "normalized_entropy": 0.916581576012616, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0956040769815445, "median": 0.09956102818250656, "p05": 0.03933291882276535, "p95": 0.16697832942008972 }, { "label": "completed_group_1", "mean": 0.09831400960683823, "median": 0.09554082155227661, "p05": 0.05786377936601639, "p95": 0.14979025721549988 }, { "label": "completed_group_2", "mean": 0.0714857280254364, "median": 0.06985784322023392, "p05": 0.04893886297941208, "p95": 0.10043824464082718 }, { "label": "completed_group_3", "mean": 0.0677608773112297, "median": 0.06590595096349716, "p05": 0.044468365609645844, "p95": 0.09555256366729736 }, { "label": "completed_group_4", "mean": 0.05654233694076538, "median": 0.054802894592285156, "p05": 0.037319183349609375, "p95": 0.08175043016672134 }, { "label": "completed_group_5", "mean": 0.05388312786817551, "median": 0.05288306996226311, "p05": 0.036485109478235245, "p95": 0.07530424743890762 }, { "label": "completed_group_6", "mean": 0.24272148311138153, "median": 0.23883119225502014, "p05": 0.1493627429008484, "p95": 0.35197100043296814 }, { "label": "completed_group_7", "mean": 0.17641445994377136, "median": 0.17445214092731476, "p05": 0.0891575887799263, "p95": 0.276222825050354 }, { "label": "current_group_partial", "mean": 0.1372738927602768, "median": 0.12184794992208481, "p05": 0.05821039155125618, "p95": 0.2647005021572113 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0349321365356445, "group": 8, "latest_source_mass": 0.13534162938594818, "layer": 31, "max_source_mass": 0.2261163890361786, "mixer_index": 60, "normalized_entropy": 0.9261375270991514, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10256819427013397, "median": 0.10616284608840942, "p05": 0.019877789542078972, "p95": 0.19627508521080017 }, { "label": "completed_group_1", "mean": 0.0938093364238739, "median": 0.09388002753257751, "p05": 0.034650709480047226, "p95": 0.15219232439994812 }, { "label": "completed_group_2", "mean": 0.0912778377532959, "median": 0.08955617249011993, "p05": 0.06106609106063843, "p95": 0.1285511702299118 }, { "label": "completed_group_3", "mean": 0.08394120633602142, "median": 0.0820196121931076, "p05": 0.05809478089213371, "p95": 0.11390723288059235 }, { "label": "completed_group_4", "mean": 0.06161237880587578, "median": 0.059880051761865616, "p05": 0.044582173228263855, "p95": 0.08485961705446243 }, { "label": "completed_group_5", "mean": 0.053358763456344604, "median": 0.052702393382787704, "p05": 0.03554790839552879, "p95": 0.07320070266723633 }, { "label": "completed_group_6", "mean": 0.2261163890361786, "median": 0.22380158305168152, "p05": 0.1451866328716278, "p95": 0.3134670853614807 }, { "label": "completed_group_7", "mean": 0.15197426080703735, "median": 0.1527099758386612, "p05": 0.0639471560716629, "p95": 0.2459363490343094 }, { "label": "current_group_partial", "mean": 0.13534162938594818, "median": 0.11519233882427216, "p05": 0.04325530678033829, "p95": 0.2904844284057617 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.046657085418701, "group": 8, "latest_source_mass": 0.09365127235651016, "layer": 31, "max_source_mass": 0.2572976052761078, "mixer_index": 61, "normalized_entropy": 0.9314737813009277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08380690217018127, "median": 0.06641843914985657, "p05": 0.039923232048749924, "p95": 0.23694062232971191 }, { "label": "completed_group_1", "mean": 0.1217331737279892, "median": 0.12164830416440964, "p05": 0.08828366547822952, "p95": 0.1561325639486313 }, { "label": "completed_group_2", "mean": 0.10242430865764618, "median": 0.10224691033363342, "p05": 0.06860878318548203, "p95": 0.13651791214942932 }, { "label": "completed_group_3", "mean": 0.09225838631391525, "median": 0.09045033901929855, "p05": 0.0655013769865036, "p95": 0.1235189214348793 }, { "label": "completed_group_4", "mean": 0.06788390874862671, "median": 0.0664045438170433, "p05": 0.047665324062108994, "p95": 0.09357043355703354 }, { "label": "completed_group_5", "mean": 0.06336672604084015, "median": 0.06275692582130432, "p05": 0.044327184557914734, "p95": 0.08375845849514008 }, { "label": "completed_group_6", "mean": 0.2572976052761078, "median": 0.25880685448646545, "p05": 0.14942821860313416, "p95": 0.36036571860313416 }, { "label": "completed_group_7", "mean": 0.11757771670818329, "median": 0.10582176595926285, "p05": 0.05813705176115036, "p95": 0.21660257875919342 }, { "label": "current_group_partial", "mean": 0.09365127235651016, "median": 0.08298523724079132, "p05": 0.04494771361351013, "p95": 0.18065699934959412 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.094015121459961, "group": 8, "latest_source_mass": 0.09723767638206482, "layer": 32, "max_source_mass": 0.2018478661775589, "mixer_index": 62, "normalized_entropy": 0.9530273523513089, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10284869372844696, "median": 0.09173205494880676, "p05": 0.03822793811559677, "p95": 0.22231025993824005 }, { "label": "completed_group_1", "mean": 0.13057854771614075, "median": 0.13079363107681274, "p05": 0.0711527168750763, "p95": 0.19180890917778015 }, { "label": "completed_group_2", "mean": 0.10152992606163025, "median": 0.09965524077415466, "p05": 0.07341106235980988, "p95": 0.13697651028633118 }, { "label": "completed_group_3", "mean": 0.10079469531774521, "median": 0.10007210820913315, "p05": 0.07479676604270935, "p95": 0.12990058958530426 }, { "label": "completed_group_4", "mean": 0.0796234980225563, "median": 0.07812212407588959, "p05": 0.05766180902719498, "p95": 0.10730832070112228 }, { "label": "completed_group_5", "mean": 0.07438632845878601, "median": 0.07310014963150024, "p05": 0.05309801548719406, "p95": 0.10082833468914032 }, { "label": "completed_group_6", "mean": 0.2018478661775589, "median": 0.20017066597938538, "p05": 0.13963642716407776, "p95": 0.2733708918094635 }, { "label": "completed_group_7", "mean": 0.1111527606844902, "median": 0.10317055881023407, "p05": 0.05416924133896828, "p95": 0.1888933926820755 }, { "label": "current_group_partial", "mean": 0.09723767638206482, "median": 0.08349506556987762, "p05": 0.03931424394249916, "p95": 0.1855989694595337 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.057201385498047, "group": 8, "latest_source_mass": 0.09480181336402893, "layer": 32, "max_source_mass": 0.24551761150360107, "mixer_index": 63, "normalized_entropy": 0.9362726990757002, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09265108406543732, "median": 0.07600092887878418, "p05": 0.04942645877599716, "p95": 0.20456968247890472 }, { "label": "completed_group_1", "mean": 0.11481083929538727, "median": 0.1135297492146492, "p05": 0.08239293098449707, "p95": 0.15063267946243286 }, { "label": "completed_group_2", "mean": 0.09007447212934494, "median": 0.0894046500325203, "p05": 0.06493785232305527, "p95": 0.11702875047922134 }, { "label": "completed_group_3", "mean": 0.08496076613664627, "median": 0.08339293301105499, "p05": 0.06288711726665497, "p95": 0.11049411445856094 }, { "label": "completed_group_4", "mean": 0.06770835816860199, "median": 0.06692098081111908, "p05": 0.0488700196146965, "p95": 0.09013402462005615 }, { "label": "completed_group_5", "mean": 0.05925413966178894, "median": 0.058933183550834656, "p05": 0.04426975175738335, "p95": 0.07536415755748749 }, { "label": "completed_group_6", "mean": 0.24551761150360107, "median": 0.24382448196411133, "p05": 0.15310630202293396, "p95": 0.3403708338737488 }, { "label": "completed_group_7", "mean": 0.15022093057632446, "median": 0.14221730828285217, "p05": 0.0825236365199089, "p95": 0.24877692759037018 }, { "label": "current_group_partial", "mean": 0.09480181336402893, "median": 0.08117073774337769, "p05": 0.04724843427538872, "p95": 0.17635159194469452 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.8435449600219727, "group": null, "latest_source_mass": 0.20717504620552063, "layer": null, "max_source_mass": 0.31774085760116577, "mixer_index": null, "normalized_entropy": 0.839033469331102, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1112452894449234, "median": 0.11452806740999222, "p05": 0.04704529047012329, "p95": 0.1625424325466156 }, { "label": "completed_group_1", "mean": 0.06898629665374756, "median": 0.06820575892925262, "p05": 0.04699244350194931, "p95": 0.09449291229248047 }, { "label": "completed_group_2", "mean": 0.05835210159420967, "median": 0.05662037432193756, "p05": 0.039408810436725616, "p95": 0.081703320145607 }, { "label": "completed_group_3", "mean": 0.042265649884939194, "median": 0.040285609662532806, "p05": 0.026811392977833748, "p95": 0.06341734528541565 }, { "label": "completed_group_4", "mean": 0.041034016758203506, "median": 0.039777740836143494, "p05": 0.027301941066980362, "p95": 0.05862141400575638 }, { "label": "completed_group_5", "mean": 0.04826357960700989, "median": 0.047516219317913055, "p05": 0.03253086656332016, "p95": 0.06693053245544434 }, { "label": "completed_group_6", "mean": 0.10493715852499008, "median": 0.09417732059955597, "p05": 0.04745320975780487, "p95": 0.19990575313568115 }, { "label": "completed_group_7", "mean": 0.31774085760116577, "median": 0.3171391487121582, "p05": 0.10361962765455246, "p95": 0.5363480448722839 }, { "label": "completed_group_8", "mean": 0.20717504620552063, "median": 0.19020375609397888, "p05": 0.03896218538284302, "p95": 0.4246461093425751 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", "optimizer_state_before": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.04804428666830063, 0.12356939166784286, 0.05923242121934891, 0.04508936405181885, 0.0799286738038063, 0.03290615975856781, 0.03170505166053772, 0.03961387276649475, 0.04508088901638985, 0.0312613770365715, 0.029994890093803406, 0.03058313950896263, 0.0497574545443058, 0.03759258985519409, 0.030690157786011696, 0.03000541403889656, 0.079553984105587, 0.0414140485227108, 0.04122982919216156, 0.05830265209078789, 0.03440643846988678, 0.033897366374731064, 0.02957344427704811, 0.03406442329287529, 0.025225287303328514, 0.01915411651134491, 0.014526215381920338, 0.011566666886210442, 0.0242468249052763, 0.013597902841866016, 0.020146220922470093, 0.02710210345685482 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.406048522767378e-05, "first_to_last_ratio": 1.2121174427344374, "imbalance_abs_log_ratio": 0.19236878289853168, "last_quartile_mean": 1.1599936385664478e-05, "mean": 1.385052358671146e-05, "normalized": [ 1.1035462446531923, 1.2453829183001823, 1.1319013131596403, 1.0360400584890102, 0.9887153549881315, 0.9324565992202007, 0.881809246461942, 0.8014211622598989, 0.7197327796271318, 0.6642367228932815, 0.63144486698815, 0.6054650987106983, 0.6612979512556375, 0.6318736594713652, 0.6080689788224007, 0.5826508433334544, 0.71610531341614, 0.685873407734308, 0.733241450138544, 0.7670951091759252, 2.8592552563843903, 2.5014923465112644, 2.101621489466362, 1.7092008525685705, 1.57037557605485, 1.4158002854110978, 1.11083466622777, 0.8417375710978994, 0.5124454971351892, 0.45441040448215947, 0.4041964735899746, 0.3902705019712373 ], "peak_layer": 21, "peak_normalized": 2.8592552563843903, "population_cv": 0.5792971940134687, "rest_mean": 1.0905006850710683e-05, "spike_contrast": 2.7286836559095273, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9756313961115665e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.5284693290595897e-05, 1.724920548440423e-05, 1.5677425835747272e-05, 1.4349697266879957e-05, 1.369422534480691e-05, 1.2915012121084146e-05, 1.2213519767101388e-05, 1.1100102710770443e-05, 9.96867584035499e-06, 9.20002639759332e-06, 8.745842023927253e-06, 8.38600863062311e-06, 9.159322871710174e-06, 8.75178102432983e-06, 8.422073733527213e-06, 8.070019248407334e-06, 9.91843353403965e-06, 9.499705811322201e-06, 1.0155777999898419e-05, 1.0624668902892154e-05, 3.960218236898072e-05, 3.464697874733247e-05, 2.910855801019352e-05, 2.3673326722928323e-05, 2.1750523956143297e-05, 1.9609575247159228e-05, 1.538564174552448e-05, 1.165850608231267e-05, 7.0976384449750185e-06, 6.293822025327245e-06, 5.598332791123539e-06, 5.405450792750344e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.784132736152969e-05, "first_to_last_ratio": 1.247704596204738, "imbalance_abs_log_ratio": 0.22130554016992907, "last_quartile_mean": 4.635819050236023e-05, "mean": 5.578696777774894e-05, "normalized": [ 1.1172279996665548, 1.2820128122064964, 1.1617993468115064, 1.0568066843619655, 1.0159130322231686, 0.9528514141416237, 0.8952059773271221, 0.8127833779089165, 0.726905325923399, 0.6687158255328219, 0.6353938715547037, 0.6079598423303039, 0.663249561462659, 0.632156813919224, 0.6072625305936415, 0.5816757612181556, 0.7067342161190907, 0.6745011066249388, 0.7095591340205663, 0.7592836625042008, 2.8322187742699385, 2.4767130392204146, 2.0818487421172924, 1.6933329620054323, 1.5551938962366243, 1.4028337924372432, 1.095017319874792, 0.8328031307001514, 0.5108569494896386, 0.4540995368567512, 0.4053097243347225, 0.3917738360059398 ], "peak_layer": 21, "peak_normalized": 2.8322187742699385, "population_cv": 0.5731363623317645, "rest_mean": 4.413512107459138e-05, "spike_contrast": 2.6896253388356377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011870693997479975, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.2326762417797e-05, 7.151960744522512e-05, 6.481326272478327e-05, 5.8956040447810665e-05, 5.667470759362914e-05, 5.315669113770127e-05, 4.994082701159641e-05, 4.534272011369467e-05, 4.055184399476275e-05, 3.730562821147032e-05, 3.5446697438601404e-05, 3.391623613424599e-05, 3.700068191392347e-05, 3.526611180859618e-05, 3.387733522686176e-05, 3.2449926948174834e-05, 3.942655894206837e-05, 3.762837150134146e-05, 3.9584152546012774e-05, 4.235813321429305e-05, 0.00015800089749973267, 0.00013816831051371992, 0.00011614002869464457, 9.446591138839722e-05, 8.675955177750438e-05, 7.825984357623383e-05, 6.108769593993202e-05, 4.645956141757779e-05, 2.8499160180217586e-05, 2.5332836230518296e-05, 2.2611000531469472e-05, 2.1855874365428463e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051294267643243074, "first_to_last_ratio": 1.247704590273494, "imbalance_abs_log_ratio": 0.22130553541620454, "last_quartile_mean": 0.04111090721562505, "mean": 0.04947244143113494, "normalized": [ 1.1172280253192406, 1.2820128195205216, 1.1617993344751307, 1.056806678091042, 1.0159130361962296, 0.9528513588115517, 0.895205957602592, 0.8127833650766507, 0.726905316511733, 0.6687157952448587, 0.6353938256699311, 0.6079598134280393, 0.6632495949619723, 0.6321568158374328, 0.6072625701847152, 0.5816757499080466, 0.7067342453085952, 0.6745011182519967, 0.7095591365309404, 0.759283697871088, 2.8322188463523386, 2.476713119523555, 2.0818486723693663, 1.6933329451602226, 1.555193919529888, 1.4028338274869276, 1.095017294552204, 0.832803082651722, 0.5108569355338682, 0.4540995509098777, 0.40530972999858744, 0.391773821129134 ], "peak_layer": 21, "peak_normalized": 2.8322188463523386, "population_cv": 0.5731363736273818, "rest_mean": 0.039139466981093086, "spike_contrast": 2.6896253725737616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10527050346136094, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.055271998047828674, 0.06342430412769318, 0.05747704952955246, 0.05228280648589134, 0.05025969818234444, 0.047139883041381836, 0.0442880243062973, 0.04021037742495537, 0.035961780697107315, 0.033083003014326096, 0.03143448382616043, 0.0300772562623024, 0.03281257674098015, 0.03127434104681015, 0.03004276193678379, 0.028776919469237328, 0.03496386855840683, 0.033369217067956924, 0.03510362282395363, 0.037563618272542953, 0.14011678099632263, 0.1225290447473526, 0.10299413651227951, 0.08377331495285034, 0.07693924009799957, 0.06940161436796188, 0.05417317897081375, 0.041200801730155945, 0.025273339822888374, 0.022465413436293602, 0.020051661878824234, 0.01938200742006302 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.2204174854323355e-06, "first_to_last_ratio": 0.735976396978067, "imbalance_abs_log_ratio": 0.306557230090771, "last_quartile_mean": 3.0169683356007226e-06, "mean": 2.8222587751258743e-06, "normalized": [ 1.1437692142315412, 0.8934875077918236, 0.843634317775851, 0.8332025934148901, 0.6691897239669076, 0.649206320818976, 0.6345003713598615, 0.6270252408101402, 0.548073507234572, 0.5298155928110359, 0.5235851424549667, 0.5213985029212593, 0.5319259781097652, 0.5365763585413705, 0.538492744687615, 0.5418387466813859, 0.5975925562684307, 0.6133961155504078, 0.6328406634942718, 0.6938494895757952, 2.792643580637443, 2.792207888188673, 2.5604162828657895, 2.199406095874773, 1.9619910931916569, 1.8148930082375383, 1.4357221480925244, 1.0849940773120685, 0.6411781557251384, 0.5785168725769138, 0.5245362458580646, 0.5100938629385489 ], "peak_layer": 21, "peak_normalized": 2.792643580637443, "population_cv": 0.7010168394314351, "rest_mean": 2.0585069512014378e-06, "spike_contrast": 3.3745422235584535, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.946518624317832e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.2280127015837934e-06, 2.5216529593308223e-06, 2.3809543563402258e-06, 2.3515133307228098e-06, 1.8886265706896666e-06, 1.8322282357985387e-06, 1.7907242408909951e-06, 1.7696274881018326e-06, 1.5468052652067854e-06, 1.495276706009463e-06, 1.4776927628190606e-06, 1.4715215002070181e-06, 1.5012327594376984e-06, 1.5143573364184704e-06, 1.5197658740362385e-06, 1.5292091575247468e-06, 1.6865608358784812e-06, 1.7311625697402633e-06, 1.7860401158031891e-06, 1.958222810571897e-06, 7.881562851252966e-06, 7.880333214416169e-06, 7.2261573222931474e-06, 6.207293154147919e-06, 5.5372465794789605e-06, 5.122097718412988e-06, 4.051979431096697e-06, 3.062134055653587e-06, 1.809570676414296e-06, 1.6327243201885722e-06, 1.4803770227445057e-06, 1.4396168808161747e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.569237398754922e-05, "first_to_last_ratio": 1.4673112290466532, "imbalance_abs_log_ratio": 0.38343163006552816, "last_quartile_mean": 3.114020603334211e-05, "mean": 4.183441066629712e-05, "normalized": [ 1.1974296567066158, 1.344289107904746, 1.230408462214659, 1.1234591616583915, 1.0623659972849115, 0.9983070144562474, 0.9335099902486312, 0.8479894563327729, 0.7756304429380606, 0.7156796904410139, 0.6795815361308393, 0.6491943107046658, 0.7109629912124332, 0.6750948063174392, 0.6474411689349395, 0.6200717213988419, 0.7662657439847727, 0.7307653188389935, 0.7545343036423542, 0.7927280102241547, 2.8287575533506253, 2.4215604626110183, 1.9807235727824308, 1.5583037155212895, 1.4117479335225582, 1.2389201568217543, 0.9856291732269274, 0.776844076198889, 0.43766749474230543, 0.39060885635754505, 0.36143095960303334, 0.352097153686139 ], "peak_layer": 21, "peak_normalized": 2.8287575533506253, "population_cv": 0.551495636352156, "rest_mean": 3.3775719169137085e-05, "spike_contrast": 2.5270030320761303, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.535134475096129e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 5.009376400266774e-05, 5.623754259431735e-05, 5.1473412895575166e-05, 4.699925193563104e-05, 4.444345540832728e-05, 4.176358561380766e-05, 3.905284029315226e-05, 3.547513915691525e-05, 3.244804247515276e-05, 2.994003807543777e-05, 2.842989306373056e-05, 2.7158661396242678e-05, 2.9742717742919922e-05, 2.8242193366168067e-05, 2.708531974349171e-05, 2.5940335035556927e-05, 3.205627581337467e-05, 3.0571136448998004e-05, 3.156549792038277e-05, 3.316330912639387e-05, 0.00011833940516225994, 0.00010130455484613776, 8.286240336019546e-05, 6.519071757793427e-05, 5.905964280827902e-05, 5.1829494623234496e-05, 4.1233215597458184e-05, 3.249881410738453e-05, 1.830956171033904e-05, 1.63408913067542e-05, 1.512025119154714e-05, 1.4729776921740267e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.5050525639235275e-05, "first_to_last_ratio": 1.5890428234254472, "imbalance_abs_log_ratio": 0.46313183711305195, "last_quartile_mean": 2.2057634396333015e-05, "mean": 3.131621542706853e-05, "normalized": [ 1.247642034455602, 1.3675088708957686, 1.2617968995203173, 1.1676693179472686, 1.0797799047255963, 1.0154085918080462, 0.9450439951530737, 0.8691124556208352, 0.8151155724977966, 0.7531778294737046, 0.7147915273060139, 0.6827252927557514, 0.7417922051240886, 0.7100839458764344, 0.679513505581043, 0.6487030016923234, 0.8192115232383504, 0.7896113859496233, 0.7975027577835305, 0.8129328118460493, 2.8268246118214386, 2.3622917424367986, 1.850934565013104, 1.4060108194738914, 1.2902617154939733, 1.123282471468924, 0.9231284485726192, 0.7885511099194735, 0.4159101173561302, 0.376982553975174, 0.35968054987980413, 0.3570178653374513 ], "peak_layer": 21, "peak_normalized": 2.8268246118214386, "population_cv": 0.5277183722759345, "rest_mean": 2.5822744107615478e-05, "spike_contrast": 2.3615213122965844, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.0980960552114996e-05, "top_five_layers": [ 21, 22, 23, 24, 2 ], "top_five_spike_overlap": 4 }, "values": [ 3.907142672687769e-05, 4.282520239939913e-05, 3.951470353058539e-05, 3.656698390841484e-05, 3.3814620110206306e-05, 3.179875420755707e-05, 2.959520134027116e-05, 2.721731289057061e-05, 2.5526334866299294e-05, 2.3586679162690416e-05, 2.238456545455847e-05, 2.138037234544754e-05, 2.3230124497786164e-05, 2.223714182036929e-05, 2.1279791326378472e-05, 2.03149229491828e-05, 2.5654604542069137e-05, 2.4727640266064554e-05, 2.4974768166430295e-05, 2.5457979063503444e-05, 8.852544851833954e-05, 7.397803710773587e-05, 5.796426557935774e-05, 4.403093771543354e-05, 4.040611383970827e-05, 3.517695586197078e-05, 2.8908889362355694e-05, 2.469443643349223e-05, 1.3024730833421927e-05, 1.1805666872533038e-05, 1.1263833584962413e-05, 1.1180448382219765e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011513851586641977, "first_to_last_ratio": 1.1690722131114868, "imbalance_abs_log_ratio": 0.15621045398792427, "last_quartile_mean": 9.848708623394486e-05, "mean": 0.00011408905686494109, "normalized": [ 1.0878105091943224, 1.2333698943486997, 1.1325669787296346, 1.0350609428916697, 0.9847532971789725, 0.9327565235282874, 0.875161061795294, 0.7921095504711437, 0.7054552974920509, 0.6480774339061697, 0.6173745394266138, 0.5959457136906282, 0.6486626275163402, 0.6167276122272274, 0.5953045899588231, 0.5685580660104519, 0.6801746911900812, 0.6508152675683995, 0.6710893308044551, 0.7221277670118063, 2.8258795357154587, 2.493070695082837, 2.1580463328702972, 1.8231222239658644, 1.644278740520026, 1.5034127662796515, 1.171932855948766, 0.8973146194608761, 0.48937000596181734, 0.4306716621780954, 0.38968515069330856, 0.3793137163819305 ], "peak_layer": 21, "peak_normalized": 2.8258795357154587, "population_cv": 0.5949686549165712, "rest_mean": 8.89708825087399e-05, "spike_contrast": 2.806841871708938, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002497271983884275, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012410727504175156, 0.00014071400801185519, 0.0001292134984396398, 0.00011808912677224725, 0.00011234957491979003, 0.00010641731205396354, 9.98463001451455e-05, 9.037103154696524e-05, 8.048472955124453e-05, 7.393854320980608e-05, 7.043567893560976e-05, 6.799088441766798e-05, 7.400530739687383e-05, 7.036187162157148e-05, 6.79177392157726e-05, 6.486625352408737e-05, 7.760048902127892e-05, 7.425090007018298e-05, 7.656394882360473e-05, 8.238687587436289e-05, 0.0003224019310437143, 0.00028443208429962397, 0.0002462094707880169, 0.00020799829508177936, 0.00018759421072900295, 0.00017152294458355755, 0.00013370471424423158, 0.00010237377864541486, 5.583176243817434e-05, 4.913492375635542e-05, 4.4458811316872016e-05, 4.327554415795021e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.07808538526296616, 0.03881331905722618, 0.01480295043438673, 0.02500341460108757, 0.015180142596364021, 0.013721407391130924, 0.016902483999729156, 0.03325798362493515, 0.026172807440161705, 0.027887089177966118, 0.03240044787526131, 0.03682815283536911, 0.029301028698682785, 0.0338764488697052, 0.0393521711230278, 0.046694882214069366, 0.029503758996725082, 0.03615153953433037, 0.030687781050801277, 0.05279085040092468, 0.024021649733185768, 0.03336174041032791, 0.04255227372050285, 0.0520954355597496, 0.03984410688281059, 0.04249191656708717, 0.05309459939599037, 0.06231515854597092, 0.06388300657272339, 0.0749494656920433, 0.10156872868537903, 0.11044161766767502 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.387307077038713e-05, "first_to_last_ratio": 1.3885610977109948, "imbalance_abs_log_ratio": 0.32826802946929257, "last_quartile_mean": 9.990968919737497e-06, "mean": 1.2854331032485788e-05, "normalized": [ 1.1519258940103256, 1.3312041641813053, 1.2167309231197612, 1.1117514471382886, 1.0587713135165344, 0.9928224052891267, 0.9274023828418168, 0.8434126323121538, 0.7336839389102785, 0.6915061646833663, 0.6596107083150127, 0.6376340360074726, 0.6952642607649582, 0.6629197998266377, 0.6345516401532968, 0.6122226131721931, 0.7648990252904823, 0.729764382313612, 0.7866676809835089, 0.8716429632898921, 2.8889885153646824, 2.3385372141657035, 1.899664892928511, 1.5404582652892223, 1.5315814746576397, 1.1978299957950485, 0.9082536044284508, 0.816087598131252, 0.48962660632970234, 0.43639989519040423, 0.42103372910248354, 0.4171498324968764 ], "peak_layer": 21, "peak_normalized": 2.8889885153646824, "population_cv": 0.5419279223583527, "rest_mean": 1.0379048506837097e-05, "spike_contrast": 2.5263256698064365, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6220856670988724e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.4807236766500864e-05, 1.7111738998210058e-05, 1.5640262063243426e-05, 1.4290821127360687e-05, 1.3609796951641329e-05, 1.2762067854055203e-05, 1.1921137229364831e-05, 1.0841505172720645e-05, 9.4310162239708e-06, 8.888849151844624e-06, 8.478854397253599e-06, 8.196358976420015e-06, 8.937156962929294e-06, 8.521390554960817e-06, 8.156736839737277e-06, 7.869712135288864e-06, 9.832265277509578e-06, 9.380632945976686e-06, 1.0112086783919949e-05, 1.120438719226513e-05, 3.7136014725547284e-05, 3.0060331482673064e-05, 2.441892138449475e-05, 1.9801560483756475e-05, 1.9687455278472044e-05, 1.5397303286590613e-05, 1.1674992492771707e-05, 1.0490260137885343e-05, 6.293822480074596e-06, 5.609628715319559e-06, 5.412106929725269e-06, 5.3621820370608475e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.7066908539127326e-05, "first_to_last_ratio": 1.4278402189707298, "imbalance_abs_log_ratio": 0.3561629661799943, "last_quartile_mean": 3.99672931052919e-05, "mean": 5.182235634038079e-05, "normalized": [ 1.1683197832075227, 1.369358043457492, 1.2477885257362629, 1.1320828936276526, 1.0861971638477104, 1.0136921452638399, 0.9398491667147963, 0.8523323061186183, 0.7376381293423032, 0.6943242114792568, 0.6618878610984286, 0.637951161315391, 0.6961169332186252, 0.6619150990682322, 0.6320240246440271, 0.6099217459665006, 0.753292310075495, 0.715605419809301, 0.7584607148457314, 0.8619138236046655, 2.8636215539276253, 2.323250279031787, 1.8821006100447468, 1.5304640359562964, 1.5177704400182797, 1.1794085853576253, 0.8974069340864187, 0.8090474514501561, 0.4887596394765995, 0.4369840848318672, 0.4222231761264232, 0.4182917472503225 ], "peak_layer": 21, "peak_normalized": 2.8636215539276253, "population_cv": 0.5384897216371255, "rest_mean": 4.2000662991389014e-05, "spike_contrast": 2.4966153616773794, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010485950042493641, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.0545084124896675e-05, 7.09633604856208e-05, 6.466334161814302e-05, 5.866720312042162e-05, 5.6289296480827034e-05, 5.2531915571307763e-05, 4.870519842370413e-05, 4.416986848809756e-05, 3.822614598902874e-05, 3.598151670303196e-05, 3.430058859521523e-05, 3.306013240944594e-05, 3.607441976782866e-05, 3.430200013099238e-05, 3.275297422078438e-05, 3.1607582059223205e-05, 3.9037382521200925e-05, 3.708435906446539e-05, 3.930522143491544e-05, 4.466640530154109e-05, 0.00014839961659163237, 0.00012039630382787436, 9.753488848218694e-05, 7.931225263746455e-05, 7.865444058552384e-05, 6.111973198130727e-05, 4.650574192055501e-05, 4.192674532532692e-05, 2.5328676201752387e-05, 2.264554495923221e-05, 2.188059988839086e-05, 2.1676863980246708e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05060750572010875, "first_to_last_ratio": 1.4278402181986947, "imbalance_abs_log_ratio": 0.356162965639293, "last_quartile_mean": 0.03544339560903609, "mean": 0.045956584566738456, "normalized": [ 1.1683197838664416, 1.3693581481609842, 1.2477884983122534, 1.1320828879264566, 1.086197129718159, 1.0136921293230559, 0.9398491347700728, 0.852332271168493, 0.7376380598562373, 0.6943242075392044, 0.6618878602537814, 0.6379511317995364, 0.6961168732850856, 0.66191505624554, 0.6320240673482492, 0.6099217105646539, 0.7532923321079824, 0.7156054155999519, 0.7584607053972269, 0.8619137716807654, 2.8636216906024767, 2.3232503851911614, 1.8821005956556751, 1.530464123018414, 1.5177704445282252, 1.1794086151290946, 0.8974069322276481, 0.8090474387174796, 0.48875963581745097, 0.43698406072332996, 0.42222316261516873, 0.41829174084974413 ], "peak_layer": 21, "peak_normalized": 2.8636216906024767, "population_cv": 0.5384897545714166, "rest_mean": 0.03724660800287017, "spike_contrast": 2.496615477158658, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0929904580116272, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.053691986948251724, 0.06293102353811264, 0.05734409764409065, 0.0520266629755497, 0.04991791024804115, 0.04658582806587219, 0.04319225624203682, 0.0391702800989151, 0.033899325877428055, 0.03190876916050911, 0.03041810542345047, 0.029318055137991905, 0.031991153955459595, 0.030419355258345604, 0.029045667499303818, 0.028029918670654297, 0.034618742763996124, 0.03288678079843521, 0.03485626354813576, 0.0396106131374836, 0.13160227239131927, 0.1067686527967453, 0.08649491518735886, 0.07033490389585495, 0.0697515457868576, 0.05420159175992012, 0.041241757571697235, 0.037181057035923004, 0.022461723536252975, 0.020082294940948486, 0.019403934478759766, 0.019223259761929512 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.187219166671639e-06, "first_to_last_ratio": 0.8437183001333121, "imbalance_abs_log_ratio": 0.16993660771354646, "last_quartile_mean": 2.5923571485009234e-06, "mean": 2.5683367930184886e-06, "normalized": [ 1.1919543817128102, 0.9792836739870964, 0.924042037649071, 0.9164810837361373, 0.7326981846944463, 0.7055637920842305, 0.6863211882787902, 0.6765290243634249, 0.5799994679651287, 0.5635357621627565, 0.5597470962238182, 0.5639314892075872, 0.5739189935534712, 0.5756380159008986, 0.5748207554502599, 0.5781987328520684, 0.6522355446992087, 0.6638052489880939, 0.6844497180630856, 0.7655738065181559, 2.843883820310657, 2.5944552048335017, 2.2989551083840256, 2.0391579131550635, 1.9644428765002167, 1.5269231604355886, 1.179167971687766, 1.0934476508277742, 0.6272800411331483, 0.5710254497958045, 0.5563528736513774, 0.556179931194537 ], "peak_layer": 21, "peak_normalized": 2.843883820310657, "population_cv": 0.6360421157910927, "rest_mean": 1.9271187023116874e-06, "spike_contrast": 3.129498185867219, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.030914482835214e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.0613402941526147e-06, 2.5151302907033823e-06, 2.3732511635898845e-06, 2.35383208746498e-06, 1.8818157059286023e-06, 1.8121254470315762e-06, 1.7627039596845862e-06, 1.7375543848174857e-06, 1.4896339735059883e-06, 1.4473496321443236e-06, 1.4376190620168927e-06, 1.4483659924735548e-06, 1.4740172673555207e-06, 1.4784322956984397e-06, 1.4763332956135855e-06, 1.4850090792606352e-06, 1.675160547165433e-06, 1.7048754443749203e-06, 1.757897393872554e-06, 1.9662513750517974e-06, 7.304051450773841e-06, 6.663434760412201e-06, 5.9044909903605e-06, 5.2372442951309495e-06, 5.045350917498581e-06, 3.921652933058795e-06, 3.028500486834673e-06, 2.808341832860606e-06, 1.6110664091684157e-06, 1.4665856724604964e-06, 1.4289015553003992e-06, 1.4284573808254208e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.514474267125479e-05, "first_to_last_ratio": 1.655519660200222, "imbalance_abs_log_ratio": 0.5041149536218862, "last_quartile_mean": 2.726922775764251e-05, "mean": 3.928292341015549e-05, "normalized": [ 1.2428531532199, 1.4224719349524337, 1.3075943550391356, 1.1908940314839356, 1.1240062299860272, 1.0512639226040563, 0.9724687394121546, 0.8822119975062471, 0.7835780621436017, 0.7378068411250925, 0.7017901972736155, 0.675554292822624, 0.7400896696370771, 0.7012250467261758, 0.6687293301443596, 0.6449026664130145, 0.8086596332888067, 0.7678950765711053, 0.7972794784799546, 0.884192918474659, 2.849023696568842, 2.282552164711599, 1.7952274549858118, 1.4143282104887471, 1.3505434814026465, 1.05691913246549, 0.8328089954359738, 0.7560703953177289, 0.41728160395883906, 0.38644818475535425, 0.376537560284359, 0.3767915423206315 ], "peak_layer": 21, "peak_normalized": 2.849023696568842, "population_cv": 0.5250932744673414, "rest_mean": 3.245689711345929e-05, "spike_contrast": 2.34598720716096, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.614346541231498e-05, "top_five_layers": [ 21, 22, 23, 2, 24 ], "top_five_spike_overlap": 4 }, "values": [ 4.8822905228007585e-05, 5.587885607383214e-05, 5.136612890055403e-05, 4.6781799028394744e-05, 4.415425064507872e-05, 4.129672015551478e-05, 3.820141500909813e-05, 3.4655866329558194e-05, 3.0781237001065165e-05, 2.898320963140577e-05, 2.7568370569497347e-05, 2.6537747544352897e-05, 2.907288580900058e-05, 2.754616980382707e-05, 2.6269643058185466e-05, 2.5333662051707506e-05, 3.176651443936862e-05, 3.016516347997822e-05, 3.131946868961677e-05, 3.4733682696241885e-05, 0.0001119179796660319, 8.966532186605036e-05, 7.0521782618016e-05, 5.555894676945172e-05, 5.305329614202492e-05, 4.151887333136983e-05, 3.271517198299989e-05, 2.9700655431952327e-05, 1.639204128878191e-05, 1.51808144437382e-05, 1.479149614169728e-05, 1.4801473298575729e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.469924399723823e-05, "first_to_last_ratio": 1.7510149859622042, "imbalance_abs_log_ratio": 0.5601956117834839, "last_quartile_mean": 1.981664593131427e-05, "mean": 2.9772959720730796e-05, "normalized": [ 1.2826990789388135, 1.4313521308000512, 1.325603166494873, 1.2222410753351367, 1.1310087328328617, 1.056511628684211, 0.9758416336275024, 0.898436085327916, 0.8214143991142518, 0.7703245121438669, 0.7298179894195511, 0.7040392340287683, 0.7681958890441348, 0.7313866125384519, 0.6969281018261846, 0.6697043803055042, 0.8568340618774071, 0.8208415080145076, 0.8328878615188123, 0.8883662275011015, 2.8312616617599446, 2.239848895619399, 1.6996416088520014, 1.290076953440798, 1.215112714243769, 0.9719493099553681, 0.8309033623504332, 0.7782938507320865, 0.3989932047584423, 0.3786265198784562, 0.3759890641537673, 0.37486854488162713 ], "peak_layer": 21, "peak_normalized": 2.8312616617599446, "population_cv": 0.5072717351311633, "rest_mean": 2.505786920260612e-05, "spike_contrast": 2.2042755539987984, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.523444851860404e-05, "top_five_layers": [ 21, 22, 23, 2, 3 ], "top_five_spike_overlap": 3 }, "values": [ 3.8189748011063784e-05, 4.261558933649212e-05, 3.9467129681725055e-05, 3.638973430497572e-05, 3.367347744642757e-05, 3.14554781652987e-05, 2.905369365180377e-05, 2.67491013801191e-05, 2.445593781885691e-05, 2.293484067195095e-05, 2.172884160245303e-05, 2.096133175655268e-05, 2.287146526214201e-05, 2.177554415538907e-05, 2.0749612303916365e-05, 1.9939081539632753e-05, 2.55104860116262e-05, 2.443888115521986e-05, 2.4797536752885208e-05, 2.6449291908647865e-05, 8.429503941442817e-05, 6.668693094979972e-05, 5.0603361160028726e-05, 3.840940917143598e-05, 3.61775018973276e-05, 2.8937807655893266e-05, 2.473845233907923e-05, 2.3172111468738876e-05, 1.1879208614118397e-05, 1.1272832125541754e-05, 1.1194307262485381e-05, 1.1160946087329648e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011354441994626541, "first_to_last_ratio": 1.3378984746048481, "imbalance_abs_log_ratio": 0.2911000803554069, "last_quartile_mean": 8.486774004268227e-05, "mean": 0.00010590371198304638, "normalized": [ 1.1416862116188895, 1.318287506150763, 1.2188213004978723, 1.1129317281051134, 1.0503469451456144, 0.9936779473327889, 0.9152461482494265, 0.8261837163725879, 0.7163781808438098, 0.6746444347634984, 0.649936107552919, 0.6258996320477875, 0.6816391399615429, 0.6448073899118816, 0.6202888218188445, 0.5944257945762098, 0.72764205396545, 0.6903920368963828, 0.7174952313478369, 0.8067894001918988, 2.8841338501609886, 2.3452358192551364, 1.972412467214386, 1.6597621298267584, 1.6194268438614279, 1.2623302767875173, 0.9611591980513982, 0.8707097795471348, 0.4633879176416832, 0.4198531088291576, 0.4079853661334264, 0.40608351533986714 ], "peak_layer": 21, "peak_normalized": 2.8841338501609886, "population_cv": 0.5599416174982532, "rest_mean": 8.440537176620974e-05, "spike_contrast": 2.630102142892712, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022199474915396422, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012090880773030221, 0.00013961154036223888, 0.0001290776999667287, 0.00011786360119003803, 0.00011123564036097378, 0.0001052341831382364, 9.692796447779983e-05, 8.749592234380543e-05, 7.586710853502154e-05, 7.144734991015866e-05, 6.883064634166658e-05, 6.62850943626836e-05, 7.218811515485868e-05, 6.828749610576779e-05, 6.569088873220608e-05, 6.295189814409241e-05, 7.70599945099093e-05, 7.311507943086326e-05, 7.598540832987055e-05, 8.544199226889759e-05, 0.000305440480588004, 0.00024836917873471975, 0.00020888580183964223, 0.00017577497055754066, 0.00017150331404991448, 0.00013368546206038445, 0.0001017903268802911, 9.221139771398157e-05, 4.9074500566348433e-05, 4.4464002712629735e-05, 4.320716470829211e-05, 4.3005751649616286e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04804428666830063, 0.18500657379627228, 0.22299833595752716, 0.2331935465335846, 0.0799286738038063, 0.09402555972337723, 0.10843002796173096, 0.12457633018493652, 0.04508088901638985, 0.0760042667388916, 0.10498681664466858, 0.13159531354904175, 0.0497574545443058, 0.07735268771648407, 0.10224150121212006, 0.13362281024456024, 0.079553984105587, 0.10536550730466843, 0.14260481297969818, 0.19080305099487305, 0.03440643846988678, 0.060867179185152054, 0.08364810794591904, 0.11119203269481659, 0.025225287303328514, 0.05530283972620964, 0.08037404716014862, 0.11502843350172043, 0.0242468249052763, 0.07164724916219711, 0.1371813863515854, 0.23144805431365967 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.406055810093676e-05, "first_to_last_ratio": 1.2121225310488972, "imbalance_abs_log_ratio": 0.19237298076220427, "last_quartile_mean": 1.1599947811191669e-05, "mean": 1.3850583115981863e-05, "normalized": [ 1.1035219335667126, 1.245398841062786, 1.1319237648162956, 1.0360361966114253, 0.9887036854141024, 0.9324573194178568, 0.8818082800629979, 0.8014300627506455, 0.7197404552580381, 0.6642341963531577, 0.63144215306254, 0.605460329509273, 0.6612975386173722, 0.6318610283296698, 0.6080617688348535, 0.5826465005104303, 0.7161183891450303, 0.6858808349031477, 0.733242501237421, 0.767094241825568, 2.859288144745241, 2.5015026079052487, 2.101601950422847, 1.7091984970083665, 1.5703665940407991, 1.4157861892600545, 1.1108502479674358, 0.8417170775086074, 0.5124528160490821, 0.4544107825414821, 0.40419536018403524, 0.39026971107747666 ], "peak_layer": 21, "peak_normalized": 2.859288144745241, "population_cv": 0.5793000123245652, "rest_mean": 1.0905047424279877e-05, "spike_contrast": 2.7286883489310063, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.975647585117258e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.5284422261174768e-05, 1.7249500160687603e-05, 1.567780418554321e-05, 1.4349705452332273e-05, 1.369412257190561e-05, 1.2915077604702674e-05, 1.2213558875373565e-05, 1.1100273695774376e-05, 9.968824997486081e-06, 9.200030945066828e-06, 8.745842023927253e-06, 8.385978617297951e-06, 9.15935652301414e-06, 8.751643690629862e-06, 8.42201006889809e-06, 8.069993782555684e-06, 9.918657269736286e-06, 9.499849511485081e-06, 1.0155836207559332e-05, 1.062470255419612e-05, 3.960280810133554e-05, 3.4647269785637036e-05, 2.9108412491041236e-05, 2.3673395844525658e-05, 2.1750493033323437e-05, 1.960946428880561e-05, 1.538592368888203e-05, 1.1658272342174314e-05, 7.0977703217067756e-06, 6.293854312389158e-06, 5.598341431323206e-06, 5.405463070928818e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.784146014775615e-05, "first_to_last_ratio": 1.2477003862842722, "imbalance_abs_log_ratio": 0.22130216603186706, "last_quartile_mean": 4.635845334632904e-05, "mean": 5.5787147289265704e-05, "normalized": [ 1.1172174922096436, 1.2820167732139756, 1.1618030425233523, 1.0568058922472814, 1.015898285952758, 0.952837653331755, 0.8952147696466347, 0.8127990870482149, 0.7269196158944218, 0.6687128912021425, 0.6353920226237901, 0.6079547558759868, 0.6632448840034066, 0.632152432145331, 0.6072547727004206, 0.5816761067060003, 0.7067464190080565, 0.6745056530323595, 0.7095637632538165, 0.7592902185109378, 2.8322130517871535, 2.4767089823758397, 2.0818425648522476, 1.6933243830446072, 1.5552023255692202, 1.402829278407864, 1.095030881819868, 0.8327931471930783, 0.5108623485290222, 0.45409986898200716, 0.4053123002322658, 0.3917743360765406 ], "peak_layer": 21, "peak_normalized": 2.8322130517871535, "population_cv": 0.5731345570971216, "rest_mean": 4.41352963712739e-05, "spike_contrast": 2.6896192391648626, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011870714224642143, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.232637679204345e-05, 7.15200585545972e-05, 6.481367745436728e-05, 5.895618596696295e-05, 5.667406730935909e-05, 5.315609450917691e-05, 4.994147820980288e-05, 4.534374238573946e-05, 4.055277167935856e-05, 3.7305584555724636e-05, 3.5446708352537826e-05, 3.391606151126325e-05, 3.7000540032749996e-05, 3.5265980841359124e-05, 3.387701144674793e-05, 3.2450050639454275e-05, 3.942736657336354e-05, 3.762874621315859e-05, 3.9584538171766326e-05, 4.2358635255368426e-05, 0.00015800108667463064, 0.00013816852879244834, 0.00011614005779847503, 9.446573676541448e-05, 8.676030120113865e-05, 7.825984357623383e-05, 6.108864909037948e-05, 4.6459153963951394e-05, 2.8499553081928752e-05, 2.533293627493549e-05, 2.2611216991208494e-05, 2.1855972590856254e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051294384989887476, "first_to_last_ratio": 1.2477003995429623, "imbalance_abs_log_ratio": 0.2213021766583686, "last_quartile_mean": 0.04111113934777677, "mean": 0.04947259998880327, "normalized": [ 1.117217517042371, 1.282016692529182, 1.1618030656626175, 1.0568059265711638, 1.0158982593254349, 0.9528376876517187, 0.8952147600149789, 0.8127990580563501, 0.7269196281255159, 0.6687128990372552, 0.6353920151588079, 0.6079547776386781, 0.6632449090716109, 0.6321524554983756, 0.6072547505273581, 0.5816761070130537, 0.706746437867777, 0.674505658209711, 0.7095637900293472, 0.7592902251062212, 2.8322130824094, 2.4767089467624213, 2.081842602529718, 1.693324355494102, 1.5552023386093694, 1.4028291808614948, 1.0950308028824434, 0.8327931847449266, 0.5108623388148443, 0.45409990274101625, 0.40531230895107145, 0.3917743350616629 ], "peak_layer": 21, "peak_normalized": 2.8322130824094, "population_cv": 0.5731345524386317, "rest_mean": 0.03913962206354848, "spike_contrast": 2.689619246048363, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10527068078517914, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.055271655321121216, 0.06342469900846481, 0.05747741833329201, 0.05228293687105179, 0.05025912821292877, 0.04713935777544975, 0.044288601726293564, 0.040211282670497894, 0.035962603986263275, 0.03308296576142311, 0.031434495002031326, 0.030077103525400162, 0.03281245008111, 0.0312742255628109, 0.03004247136414051, 0.028777029365301132, 0.034964583814144135, 0.03336954861879349, 0.03510396555066109, 0.03756406158208847, 0.14011694490909576, 0.12252923101186752, 0.1029941663146019, 0.0837731584906578, 0.0769399031996727, 0.06940160691738129, 0.054174020886421204, 0.04120044410228729, 0.02527368813753128, 0.022465502843260765, 0.020051853731274605, 0.019382094964385033 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.2204297493999547e-06, "first_to_last_ratio": 0.7359606295430359, "imbalance_abs_log_ratio": 0.3065786541527352, "last_quartile_mean": 3.0170496359005483e-06, "mean": 2.8223206918198684e-06, "normalized": [ 1.1437255119946472, 0.8934281083094181, 0.843667370045445, 0.833234424409991, 0.6691350034907739, 0.6491575975670105, 0.6344872974587108, 0.6270766601877794, 0.5480904457549984, 0.5298089242035497, 0.5236022556745512, 0.5213819889179987, 0.5319070176791533, 0.5365159271613865, 0.5384760973551012, 0.5417839198031603, 0.5976312076631317, 0.6133914803428778, 0.632878138794897, 0.6938486884695898, 2.7927450515890637, 2.792273921126163, 2.5604223062656137, 2.1993623563278324, 1.9620758229843447, 1.8148751057591102, 1.435693067783232, 1.0849680187180202, 0.6411889832806804, 0.5785351975720235, 0.5245373868045311, 0.5100947165052133 ], "peak_layer": 21, "peak_normalized": 2.7927450515890637, "population_cv": 0.7010322041721825, "rest_mean": 2.058529688568982e-06, "spike_contrast": 3.374637804812921, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.946792109374656e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.227960178264766e-06, 2.5215406367351534e-06, 2.3810998754925095e-06, 2.3516547571489355e-06, 1.8885135659729713e-06, 1.832130919865449e-06, 1.7907266283145873e-06, 1.7698114334052661e-06, 1.5468870060431072e-06, 1.4952906894905027e-06, 1.477773480473843e-06, 1.4715071756654652e-06, 1.5012121821200708e-06, 1.5142200027185027e-06, 1.519752231615712e-06, 1.5290879673557356e-06, 1.6867069234649534e-06, 1.7311874671577243e-06, 1.7861850665212842e-06, 1.958263510459801e-06, 7.882022146077361e-06, 7.880692464823369e-06, 7.22633285477059e-06, 6.207305887073744e-06, 5.5376071941282135e-06, 5.1221595640527084e-06, 4.0519862523069605e-06, 3.0621276891906746e-06, 1.809640934880008e-06, 1.6328118590536178e-06, 1.4804127204115503e-06, 1.4396508731806534e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.56924158243055e-05, "first_to_last_ratio": 1.4673115279536288, "imbalance_abs_log_ratio": 0.38343183377619244, "last_quartile_mean": 3.1140228202275466e-05, "mean": 4.183447202876778e-05, "normalized": [ 1.1974206825409346, 1.3442935712461421, 1.2304058748098903, 1.1234622096860003, 1.0623591343774659, 0.9983032021931795, 0.9335117515893174, 0.8479976043257199, 0.7756355664609215, 0.7156793363778972, 0.6795817133031917, 0.6491928801846335, 0.7109607744004338, 0.6750928595197824, 0.6474397844670364, 0.6200735511625727, 0.7662731422367131, 0.7307687689442663, 0.7545377188834692, 0.7927345870100259, 2.8287570565248865, 2.421556562840323, 1.9807276243744367, 1.558292733686825, 1.4117490803489046, 1.238912861023533, 0.9856350322605589, 0.7768378929765343, 0.43766980945827866, 0.39060663115145766, 0.36143264697135674, 0.3520973546633113 ], "peak_layer": 21, "peak_normalized": 2.8287570565248865, "population_cv": 0.551494708887264, "rest_mean": 3.377578448433067e-05, "spike_contrast": 2.5269993301954115, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.535138476872817e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 5.009346205042675e-05, 5.623781180474907e-05, 5.14733801537659e-05, 4.699944838648662e-05, 4.444323349162005e-05, 4.176348738837987e-05, 3.905297126038931e-05, 3.547553205862641e-05, 3.244830440962687e-05, 2.9940067179268226e-05, 2.8429942176444456e-05, 2.715864138735924e-05, 2.9742668630206026e-05, 2.824215334840119e-05, 2.7085301553597674e-05, 2.594044963188935e-05, 3.205663233529776e-05, 3.057132562389597e-05, 3.156568709528074e-05, 3.31636329065077e-05, 0.00011833955795736983, 0.00010130454029422253, 8.286269439850003e-05, 6.519035378005356e-05, 5.9059777413494885e-05, 5.182926543056965e-05, 4.123352118767798e-05, 3.249860310461372e-05, 1.830968540161848e-05, 1.6340822185156867e-05, 1.5120343960006721e-05, 1.4729806935065426e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.505123777358676e-05, "first_to_last_ratio": 1.5889493494930538, "imbalance_abs_log_ratio": 0.4630730113341737, "last_quartile_mean": 2.2059380171413068e-05, "mean": 3.131689976498819e-05, "normalized": [ 1.2479593211749789, 1.367487352044708, 1.261680923885286, 1.1674855830446973, 1.0794756507539984, 1.0156514953231304, 0.945160188314607, 0.8690478102601445, 0.8149681766862981, 0.7530914967772463, 0.7148635553828698, 0.6827279730652124, 0.7417389963838543, 0.7102014979770842, 0.6794125192711595, 0.6486601330546842, 0.8192809210261347, 0.7895373258521271, 0.7975483511256357, 0.812938977938045, 2.826829287271584, 2.3623544295272914, 1.8510256189591003, 1.4057347513501408, 1.2902085448265628, 1.123507683724877, 0.9230685473509727, 0.7887132397564032, 0.4159196155359273, 0.3769758263062851, 0.35976736593467973, 0.35697684011427405 ], "peak_layer": 21, "peak_normalized": 2.826829287271584, "population_cv": 0.5277213122295441, "rest_mean": 2.5823506533655477e-05, "spike_contrast": 2.3614617610009216, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.0981223214184865e-05, "top_five_layers": [ 21, 22, 23, 24, 2 ], "top_five_spike_overlap": 4 }, "values": [ 3.908221697201952e-05, 4.282546433387324e-05, 3.9511935028713197e-05, 3.656202898127958e-05, 3.380583075340837e-05, 3.180705607519485e-05, 2.9599486879305914e-05, 2.7215883164899424e-05, 2.5522276700939983e-05, 2.358449091843795e-05, 2.238731030956842e-05, 2.1380923499236815e-05, 2.3228965801536106e-05, 2.2241309125092812e-05, 2.1277093765093014e-05, 2.031402436841745e-05, 2.5657338483142667e-05, 2.4725861294427887e-05, 2.4976741769933142e-05, 2.5458728487137705e-05, 8.85275294422172e-05, 7.398161687888205e-05, 5.796838377136737e-05, 4.402325430419296e-05, 4.0405331674264744e-05, 3.518477751640603e-05, 2.890764517360367e-05, 2.4700053472770378e-05, 1.3025312910031062e-05, 1.180571416625753e-05, 1.1266798537690192e-05, 1.1179407920280937e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011513973367982544, "first_to_last_ratio": 1.1689761361783517, "imbalance_abs_log_ratio": 0.15612826840504843, "last_quartile_mean": 9.849622256297152e-05, "mean": 0.00011409382295823889, "normalized": [ 1.087599643787199, 1.2332404432021893, 1.1325955558216252, 1.0348100003758545, 0.9846631201892967, 0.9329597002283465, 0.8753802278299959, 0.7920881953113688, 0.705656362899583, 0.6479209049109275, 0.6172280934926675, 0.5959629720755263, 0.6485619380890737, 0.616624813163851, 0.5952030046676504, 0.5686106501058988, 0.6801552059812119, 0.6508138445342289, 0.6710903132095474, 0.7219939084470286, 2.8256801162653837, 2.4931864357424143, 2.15835309890828, 1.8232898013061003, 1.6443057108717645, 1.5033517491402877, 1.1720956223235564, 0.8974197290603315, 0.48941489739710603, 0.4307568860388074, 0.3896112863482277, 0.3793757682746689 ], "peak_layer": 21, "peak_normalized": 2.8256801162653837, "population_cv": 0.5949861940504896, "rest_mean": 8.897283448647984e-05, "spike_contrast": 2.80700465650208, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00024974716070573776, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012408840120770037, 0.00014070511679165065, 0.00012922215682920069, 0.00011806542897829786, 0.00011234397970838472, 0.0001064449388650246, 9.987547673517838e-05, 9.037237032316625e-05, 8.05110321380198e-05, 7.392377301584929e-05, 7.042191282380372e-05, 6.799569382565096e-05, 7.399691094178706e-05, 7.035308226477355e-05, 6.790898623876274e-05, 6.487496284535155e-05, 7.76015076553449e-05, 7.425383955705911e-05, 7.656725938431919e-05, 8.237504516728222e-05, 0.0003223926469217986, 0.00028445717180147767, 0.0002462547563482076, 0.00020802610379178077, 0.00018760512466542423, 0.00017152314831037074, 0.00013372887042351067, 0.00010239004768664017, 5.583921665675007e-05, 4.9146699893753976e-05, 4.445224112714641e-05, 4.328443174017593e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.09451264888048172, 0.2031717449426651, 0.22856928408145905, 0.2430499792098999, 0.08243431895971298, 0.09709372371435165, 0.11282631754875183, 0.13645486533641815, 0.05442482605576515, 0.0885990634560585, 0.12215005606412888, 0.15440259873867035, 0.060873307287693024, 0.09471380710601807, 0.1260664463043213, 0.163865864276886, 0.08853588253259659, 0.12131853401660919, 0.15660560131072998, 0.2097783088684082, 0.04455159604549408, 0.07411240041255951, 0.10008715838193893, 0.13061410188674927, 0.04960336163640022, 0.07874798029661179, 0.11358420550823212, 0.1580832600593567, 0.06955241411924362, 0.13302887976169586, 0.22457914054393768, 0.32859006524086 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3873141369913355e-05, "first_to_last_ratio": 1.3885721694739201, "imbalance_abs_log_ratio": 0.32827600298864523, "last_quartile_mean": 9.990940100124135e-06, "mean": 1.2854364840109156e-05, "normalized": [ 1.1519312840854947, 1.3311758992301272, 1.2167315437597699, 1.1117686172453958, 1.058772632613286, 0.9928324590403058, 0.927417632168206, 0.8434123244463263, 0.7336779055664342, 0.6915156665898262, 0.6596055773251193, 0.6376390806056413, 0.6952626444464219, 0.6629130327970861, 0.6345504665270513, 0.6122283613875565, 0.7648834288449639, 0.7297709534484661, 0.7866551404509706, 0.8716621799696268, 2.8889843133608784, 2.338532478770988, 1.899676736110628, 1.5404711947102903, 1.5315734843108408, 1.1978159493624418, 0.9082426544704836, 0.8160818433380076, 0.4896280426071342, 0.4364018959793389, 0.4210420673902388, 0.4171425090406535 ], "peak_layer": 21, "peak_normalized": 2.8889843133608784, "population_cv": 0.5419273520370343, "rest_mean": 1.037907206948687e-05, "spike_contrast": 2.5263285220415503, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6220945801469498e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.4807344996370375e-05, 1.7111420675064437e-05, 1.5640311175957322e-05, 1.429107942385599e-05, 1.3609849702334031e-05, 1.2762230653606821e-05, 1.1921364603040274e-05, 1.0841529729077592e-05, 9.430963473278098e-06, 8.888994670996908e-06, 8.478810741507914e-06, 8.196445378416684e-06, 8.937159691413399e-06, 8.521325980836991e-06, 8.15674320620019e-06, 7.869806722737849e-06, 9.832090654526837e-06, 9.380742085340898e-06, 1.0111952178704087e-05, 1.120466367865447e-05, 3.713605838129297e-05, 3.00603496725671e-05, 2.4419137844233774e-05, 1.98017787624849e-05, 1.9687404346768744e-05, 1.539716322440654e-05, 1.1674882443912793e-05, 1.0490213753655553e-05, 6.293857495620614e-06, 5.609669187833788e-06, 5.4122283472679555e-06, 5.3621020015270915e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.706737192667788e-05, "first_to_last_ratio": 1.4278557447018398, "imbalance_abs_log_ratio": 0.3561738396982594, "last_quartile_mean": 3.996718305643299e-05, "mean": 5.182250379220932e-05, "normalized": [ 1.1683241108409874, 1.3693593420455452, 1.2478029467661282, 1.1320859203517522, 1.0862154844943164, 1.0136862423485675, 0.9398593392770029, 0.8523331101914637, 0.7376332926932397, 0.6943335382230071, 0.6618887156422048, 0.6379567874173152, 0.6961101086882286, 0.6619092142621851, 0.6320190672968202, 0.6099299790483219, 0.7532799174046017, 0.7156061917099562, 0.758449009476584, 0.8619189528616946, 2.863607509132394, 2.3232710469251585, 1.8820910428129736, 1.530471615417879, 1.517753064129064, 1.1793980690808934, 0.8973983434078874, 0.8090471852696426, 0.4887601793176453, 0.43698385938136236, 0.4222309955613731, 0.41828581852380436 ], "peak_layer": 21, "peak_normalized": 2.863607509132394, "population_cv": 0.5384894543542103, "rest_mean": 4.200080675892187e-05, "spike_contrast": 2.4966108002124767, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010485966777196154, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.054548066458665e-05, 7.096362969605252e-05, 6.466427294071764e-05, 5.866752690053545e-05, 5.6290406064363196e-05, 5.253175913821906e-05, 4.870586417382583e-05, 4.417003583512269e-05, 3.822600410785526e-05, 3.59821024176199e-05, 3.430073047638871e-05, 3.306051803519949e-05, 3.607416874729097e-05, 3.430179276620038e-05, 3.275281051173806e-05, 3.160809865221381e-05, 3.903685137629509e-05, 3.708450458361767e-05, 3.9304726669797674e-05, 4.4666798203252256e-05, 0.00014839931100141257, 0.00012039772263960913, 9.753467020345852e-05, 7.931287109386176e-05, 7.865376392146572e-05, 6.111936090746894e-05, 4.65054290543776e-05, 4.1926850826712325e-05, 2.5328776246169582e-05, 2.2645597709924914e-05, 2.1881067368667573e-05, 2.167661841667723e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05060791550204158, "first_to_last_ratio": 1.4278557661609772, "imbalance_abs_log_ratio": 0.3561738547271846, "last_quartile_mean": 0.035443296656012535, "mean": 0.045956714369822294, "normalized": [ 1.1683241037091443, 1.3693593062308924, 1.2478029694871473, 1.1320859320845278, 1.0862154618577222, 1.013686266934737, 0.9398592878142475, 0.8523331062206766, 0.7376333014160883, 0.6943335139035525, 0.6618887468442457, 0.6379567875276879, 0.6961101245379498, 0.6619092147086973, 0.632019080315496, 0.6099299583472245, 0.75327990973307, 0.7156062315279661, 0.7584489979720741, 0.861918956956209, 2.863607441781645, 2.3232710596995285, 1.8820912266878924, 1.5304716351616434, 1.5177530257827039, 1.179397988455519, 0.8973983179704046, 0.8090471801133733, 0.48876020079385396, 0.43698383973821214, 0.4222310083457578, 0.41828581734010906 ], "peak_layer": 21, "peak_normalized": 2.863607441781645, "population_cv": 0.538489454771283, "rest_mean": 0.037246735076661465, "spike_contrast": 2.496610840158122, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09299060255289078, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.05369233712553978, 0.06293125450611115, 0.05734492465853691, 0.05202694982290268, 0.04991889372467995, 0.04658569023013115, 0.043192844837903976, 0.03917042911052704, 0.033899202942848206, 0.031909286975860596, 0.030418232083320618, 0.029318397864699364, 0.031990934163331985, 0.03041917271912098, 0.02904552035033703, 0.028030376881361008, 0.03461826965212822, 0.03288691118359566, 0.03485582396388054, 0.03961096331477165, 0.1316019892692566, 0.10676990449428558, 0.08649472892284393, 0.07033544778823853, 0.06975094228982925, 0.05420125648379326, 0.04124147817492485, 0.037181150168180466, 0.02246181294322014, 0.020082341507077217, 0.019404349848628044, 0.019223041832447052 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.187229483752162e-06, "first_to_last_ratio": 0.843721752674449, "imbalance_abs_log_ratio": 0.16993251566757275, "last_quartile_mean": 2.592358768538361e-06, "mean": 2.5683487088201673e-06, "normalized": [ 1.191975321874382, 0.9792375219295126, 0.9240659028379828, 0.9165026822465744, 0.7327411746199551, 0.7055071798348216, 0.68634443005073, 0.6764996809984228, 0.5799709708281261, 0.5635565193402196, 0.5596940376847175, 0.5639570693866405, 0.5739412518179093, 0.5756289711426894, 0.5747974170292999, 0.5782423953100819, 0.652186394989175, 0.6637611802901628, 0.6844201166273622, 0.7655503355998897, 2.843845838013164, 2.5945154076809365, 2.2989812705463573, 2.039289390897498, 1.9644018920207635, 1.5269105875005642, 1.1791879088220734, 1.0934639133198125, 0.6272666844428548, 0.5710249694984988, 0.5563916798315591, 0.5561399029872623 ], "peak_layer": 21, "peak_normalized": 2.843845838013164, "population_cv": 0.6360528130844023, "rest_mean": 1.9271144327393374e-06, "spike_contrast": 3.129556655898081, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.031013799656648e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.061408278881572e-06, 2.515023425075924e-06, 2.3733234684186755e-06, 2.3538984805782093e-06, 1.8819348497345345e-06, 1.8119884543921216e-06, 1.762771830726706e-06, 1.7374870822095545e-06, 1.4895676940795965e-06, 1.4474096587946406e-06, 1.43748945902189e-06, 1.4484384109891835e-06, 1.474081273045158e-06, 1.4784159247938078e-06, 1.4762802038603695e-06, 1.4851281093797297e-06, 1.675042085480527e-06, 1.7047701703631901e-06, 1.7578295228304341e-06, 1.9662002159748226e-06, 7.303987786144717e-06, 6.663620297331363e-06, 5.904585577809485e-06, 5.237606274022255e-06, 5.0452690629754215e-06, 3.921638835890917e-06, 3.0285657430795254e-06, 2.8083966299163876e-06, 1.6110395790747134e-06, 1.4665912431155448e-06, 1.429007852493669e-06, 1.4283612017607084e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.514497459240374e-05, "first_to_last_ratio": 1.6555269986325893, "imbalance_abs_log_ratio": 0.5041193863183254, "last_quartile_mean": 2.7269246970718086e-05, "mean": 3.9283020413449776e-05, "normalized": [ 1.2428521215956514, 1.4224732380710257, 1.3076049249463846, 1.1908965547141857, 1.1240104927458778, 1.0512648458306981, 0.9724745802930264, 0.8822121342574466, 0.7835766828764287, 0.7378119649346744, 0.7017902701963611, 0.6755581812134592, 0.7400830264097682, 0.701223315160251, 0.6687269842506862, 0.6449060748365787, 0.8086534690047941, 0.7678965143140377, 0.7972719531557511, 0.8841907350981015, 2.849012586533336, 2.2825637536620915, 1.7952254297971346, 1.4143290706697607, 1.3505415355877917, 1.056910410344958, 0.832807309381792, 0.7560735292299187, 0.4172799715855694, 0.3864473693959841, 0.37654214074567177, 0.3767888291608037 ], "peak_layer": 21, "peak_normalized": 2.849012586533336, "population_cv": 0.5250931691969987, "rest_mean": 3.245698109013675e-05, "spike_contrast": 2.345986293299445, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.61436327593401e-05, "top_five_layers": [ 21, 22, 23, 2, 24 ], "top_five_spike_overlap": 4 }, "values": [ 4.882298526354134e-05, 5.587904524873011e-05, 5.136667095939629e-05, 4.678201366914436e-05, 4.415452713146806e-05, 4.1296858398709446e-05, 3.820173878921196e-05, 3.465595727902837e-05, 3.078125882893801e-05, 2.8983482479816303e-05, 2.7568441510084085e-05, 2.6537965823081322e-05, 2.9072696634102613e-05, 2.754616980382707e-05, 2.6269615773344412e-05, 2.533385850256309e-05, 3.17663507303223e-05, 3.0165294447215274e-05, 3.1319250410888344e-05, 3.4733682696241885e-05, 0.00011191781959496439, 8.966599853010848e-05, 7.052187720546499e-05, 5.5559117754455656e-05, 5.3053350711707026e-05, 4.1518633224768564e-05, 3.271518653491512e-05, 2.970085188280791e-05, 1.6392017641919665e-05, 1.518081990070641e-05, 1.4791712601436302e-05, 1.4801403267483693e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.469784724074998e-05, "first_to_last_ratio": 1.7508907324210148, "imbalance_abs_log_ratio": 0.5601246483988392, "last_quartile_mean": 1.9817254496956593e-05, "mean": 2.9772389552817913e-05, "normalized": [ 1.2825657703659485, 1.4312331552120288, 1.325820640442136, 1.2221823686225868, 1.130955243942597, 1.0564145565086833, 0.9757410003641811, 0.8985840377367545, 0.8214467481888739, 0.7702253806339946, 0.7297568784639149, 0.7040532057825352, 0.7682184821342395, 0.7314383768963997, 0.6969099228187515, 0.6697241707472397, 0.8568233441617242, 0.8208320560978474, 0.8328912261837936, 0.8882556709751984, 2.8316971252422305, 2.2395765326860197, 1.6996117178665944, 1.2900403186629539, 1.2152436368088633, 0.9720044593739222, 0.8310137912106149, 0.7782207156702814, 0.3990111711474371, 0.3786791961781316, 0.37596186733999515, 0.3748672315335271 ], "peak_layer": 21, "peak_normalized": 2.8316971252422305, "population_cv": 0.507296052444767, "rest_mean": 2.505713847414073e-05, "spike_contrast": 2.204351683440534, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5234745377674696e-05, "top_five_layers": [ 21, 22, 23, 2, 3 ], "top_five_spike_overlap": 3 }, "values": [ 3.818504774244502e-05, 4.2611231037881225e-05, 3.94728485844098e-05, 3.638728958321735e-05, 3.367124008946121e-05, 3.145198570564389e-05, 2.9050141165498644e-05, 2.675299401744269e-05, 2.4456432583974674e-05, 2.2931450075702742e-05, 2.1726606064476073e-05, 2.096134630846791e-05, 2.2871699911775067e-05, 2.1776668290840462e-05, 2.0748673705384135e-05, 1.9939288904424757e-05, 2.5509678380331025e-05, 2.44381317315856e-05, 2.4797162041068077e-05, 2.644549385877326e-05, 8.43063899083063e-05, 6.667754496447742e-05, 5.06015021528583e-05, 3.840758290607482e-05, 3.618070695665665e-05, 2.8938895411556587e-05, 2.4741266315686516e-05, 2.3169490305008367e-05, 1.1879516023327596e-05, 1.127418454416329e-05, 1.1193283171451185e-05, 1.1160693247802556e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011355107744748238, "first_to_last_ratio": 1.3380154347456796, "imbalance_abs_log_ratio": 0.2911874973238657, "last_quartile_mean": 8.486529713991331e-05, "mean": 0.00010590487272565952, "normalized": [ 1.14177249304275, 1.318239805267508, 1.2190343862407116, 1.1130801571737208, 1.0501716457006294, 0.9937227743286174, 0.9154089043884999, 0.8261602336216923, 0.7161833889440711, 0.6746260480574641, 0.6498212581625004, 0.6259761084931398, 0.6817894792861978, 0.6448396206511546, 0.6202148320012568, 0.5943358056681826, 0.7276123000587434, 0.6905927768196939, 0.7174064355718048, 0.8068892453627287, 2.8834514867746446, 2.3459559521323445, 1.9722659474769593, 1.659767709595541, 1.6192051847503666, 1.2622818151535096, 0.9611298389601171, 0.8706981752777411, 0.4632426508072287, 0.41992689699434765, 0.4080288833982593, 0.4061677598378732 ], "peak_layer": 21, "peak_normalized": 2.8834514867746446, "population_cv": 0.5599097168924846, "rest_mean": 8.44075709917686e-05, "spike_contrast": 2.6299809303873802, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022199030208867044, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012091927055735141, 0.00013960801879875362, 0.00012910168152302504, 0.00011788061237893999, 0.00011121829447802156, 0.0001052400839398615, 9.69462635111995e-05, 8.749439439270645e-05, 7.584731065435335e-05, 7.14461857569404e-05, 6.881923764012754e-05, 6.62939200992696e-05, 7.220482802949846e-05, 6.82916579535231e-05, 6.56837728456594e-05, 6.294305785559118e-05, 7.70576880313456e-05, 7.313714013434947e-05, 7.597683725180104e-05, 8.545350283384323e-05, 0.00030537156271748245, 0.00024844816653057933, 0.00020887257414869964, 0.00017577748803887516, 0.00017148171900771558, 0.0001336817949777469, 0.00010178833326790482, 9.221117943525314e-05, 4.905965397483669e-05, 4.447230458026752e-05, 4.321224696468562e-05, 4.30151449108962e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05595625191926956, 0.0632273480296135, 0.11160287261009216, 0.12218338996171951, 0.12365243583917618, 0.0926642045378685, 0.093480683863163, 0.08749273419380188, 0.09927791357040405, 0.07820390164852142, 0.07669360190629959, 0.07840896397829056, 0.08920575678348541, 0.06850729882717133, 0.07440610229969025, 0.07867594808340073, 0.09058429300785065, 0.07234266400337219, 0.07771008461713791, 0.08416424691677094, 0.08947562426328659, 0.056279782205820084, 0.057985786348581314, 0.06238175556063652, 0.06515856087207794, 0.05041429027915001, 0.05083387345075607, 0.05600840970873833, 0.06128782033920288, 0.056301049888134, 0.06424970924854279, 0.06844019144773483 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.178191703540506e-06, "first_to_last_ratio": 2.0927096232778837, "imbalance_abs_log_ratio": 0.7384596967524485, "last_quartile_mean": 4.385793232586366e-06, "mean": 9.837752486419049e-06, "normalized": [ 1.3396520059806467, 1.2909822759863991, 0.8248929730174986, 0.8734689592802515, 0.4594300774035242, 0.8015983028871521, 0.902358022333612, 0.9712665986901348, 0.6517279620702138, 0.5032471363266136, 0.3886238838550298, 0.3309759496712648, 0.3194854781922982, 0.5459377912807375, 0.43221827830345944, 0.4611789434475917, 1.0400100603479894, 1.0312967006898324, 1.4985429159426815, 1.6347346226131838, 2.6918571666331577, 4.297443405515621, 2.6798531616309957, 2.4627171634441143, 1.010352002858167, 0.8524077751550129, 0.3921416774715023, 0.37035603642720544, 0.29698661283582345, 0.19544726726737693, 0.19990393116510585, 0.24890486127580272 ], "peak_layer": 22, "peak_normalized": 4.297443405515621, "population_cv": 0.8934604862364398, "rest_mean": 6.871042353816813e-06, "spike_contrast": 3.763328163463758, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5857987202471124e-05, "top_five_layers": [ 22, 21, 23, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.3179164852772374e-05, 1.2700364095508121e-05, 8.115092896332499e-06, 8.592971425969154e-06, 4.5197593863122165e-06, 7.88592569733737e-06, 8.877174877852667e-06, 9.555080396239646e-06, 6.411538379325066e-06, 4.950820766680408e-06, 3.823185579676647e-06, 3.2560594718233915e-06, 3.1430190574610606e-06, 5.370800863602199e-06, 4.252056442055618e-06, 4.536964297585655e-06, 1.0231361557089258e-05, 1.014564168144716e-05, 1.4742294297320768e-05, 1.6082114598248154e-05, 2.6481824534130283e-05, 4.227718454785645e-05, 2.636373210407328e-05, 2.4227601898019202e-05, 9.939592928276397e-06, 8.385776709474158e-06, 3.8577927625738084e-06, 3.6434710182220442e-06, 2.9216807888587937e-06, 1.922761839523446e-06, 1.9666053958644625e-06, 2.4486644178978167e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.4749664109767764e-05, "first_to_last_ratio": 2.002663558914822, "imbalance_abs_log_ratio": 0.6944780739856734, "last_quartile_mean": 1.735172338612756e-05, "mean": 3.858046457594355e-05, "normalized": [ 0.8375866313273315, 1.2899140602905792, 0.8694344042805302, 0.9158393863926719, 0.4754536346534661, 0.8638099380048725, 0.9821647500595908, 0.9714468918970042, 0.6838803886511606, 0.5360010655021746, 0.3851151654093992, 0.3481231986646324, 0.3212694297422333, 0.5688836414280135, 0.4682021401391152, 0.46212538368136635, 1.0036599670842112, 1.0844213697059686, 1.4666068093127032, 1.6452318694807484, 2.683098703810988, 4.385873690297744, 2.767792621757238, 2.3860317964922024, 1.0350167413971205, 0.871092691603006, 0.39147360723073643, 0.341414306210179, 0.30371057059916196, 0.20069739154416044, 0.20302718339269, 0.25160056995699964 ], "peak_layer": 22, "peak_normalized": 4.385873690297744, "population_cv": 0.8995046127848129, "rest_mean": 2.6780824454261972e-05, "spike_contrast": 3.8198421190406635, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010229852123302407, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 3.2314481359208e-05, 4.9765483709052205e-05, 3.354318323545158e-05, 3.533350900397636e-05, 1.8343222109251656e-05, 3.332618871354498e-05, 3.7892372347414494e-05, 3.7478872400242835e-05, 2.6384423108538613e-05, 2.0679170120274648e-05, 1.4857921996735968e-05, 1.3430754734145012e-05, 1.239472385350382e-05, 2.194779517594725e-05, 1.8063456082018092e-05, 1.7829011994763277e-05, 3.8721667806385085e-05, 4.183748023933731e-05, 5.658237205352634e-05, 6.34738098597154e-05, 0.00010351519449613988, 0.00016920904454309493, 0.00010678272519726306, 9.205421520164236e-05, 3.9931426726980135e-05, 3.3607160730753094e-05, 1.5103233636182267e-05, 1.3171922546462156e-05, 1.171729491034057e-05, 7.742998604953755e-06, 7.832883056835271e-06, 9.706866876513232e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.030816350365057588, "first_to_last_ratio": 2.002663649499559, "imbalance_abs_log_ratio": 0.6944781192178017, "last_quartile_mean": 0.015387681487482041, "mean": 0.03421354205056559, "normalized": [ 0.8375866167785538, 1.2899141113192616, 0.8694344424609419, 0.915839393595925, 0.475453642772802, 0.8638099562451347, 0.9821647993180688, 0.9714468517291538, 0.6838804035566444, 0.5360010365403223, 0.3851151782472681, 0.3481232059212625, 0.321269430974011, 0.5688836365906816, 0.46820216481780275, 0.4621253765317527, 1.0036599271213709, 1.0844213253993487, 1.4666067752392928, 1.645231903565359, 2.6830987807805085, 4.385873655623207, 2.767792506758732, 2.3860319203463485, 1.0350167426483041, 0.8710926293236593, 0.3914735937130686, 0.34141432168112545, 0.3037105506770517, 0.20069737952502648, 0.20302717344287544, 0.25160056675513426 ], "peak_layer": 22, "peak_normalized": 4.385873655623207, "population_cv": 0.8995046133337352, "rest_mean": 0.023749503048343792, "spike_contrast": 3.8198421448186797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09071935266256333, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.02865680493414402, 0.0441325306892395, 0.02974643185734749, 0.03133410960435867, 0.016266953200101852, 0.02955399826169014, 0.03360333666205406, 0.03323663771152496, 0.02339797094464302, 0.01833849400281906, 0.013176154345273972, 0.01191052794456482, 0.010991765186190605, 0.019463524222373962, 0.016018854454159737, 0.015810946002602577, 0.03433876112103462, 0.03710189461708069, 0.050177812576293945, 0.05628921091556549, 0.0917983129620552, 0.15005627274513245, 0.09469598531723022, 0.08163460344076157, 0.035411588847637177, 0.029803164303302765, 0.013393698260188103, 0.011680993251502514, 0.010391013696789742, 0.006866568233817816, 0.006946278735995293, 0.008608146570622921 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8313342426145027e-06, "first_to_last_ratio": 0.9841100436889928, "imbalance_abs_log_ratio": 0.01601755516546022, "last_quartile_mean": 2.87705044854647e-06, "mean": 2.8823285838086576e-06, "normalized": [ 3.860850182514526, 1.3779145113400497, 0.37073650468044217, 0.4054583128312923, 0.3411145989186331, 0.46814184382330143, 0.46288850930954656, 0.5713590261909561, 0.3765635727895922, 0.3810005221832118, 0.35155981621084625, 0.393783350864445, 0.33878649322165827, 0.47478509983905887, 0.4584607500663275, 0.5226177027680562, 0.4037194811557118, 0.6360221076907124, 0.6562906885789336, 0.9724019253458765, 1.6772106521914107, 3.0942458588650434, 2.9339099985078207, 2.4848281321882983, 1.7824259625811416, 1.6267190981510171, 1.2975773719544916, 1.1633314414023308, 0.5790326834473365, 0.5326514143129327, 0.49977516979896786, 0.503837216276031 ], "peak_layer": 1, "peak_normalized": 3.860850182514526, "population_cv": 0.9094699254837627, "rest_mean": 2.1379810404040554e-06, "spike_contrast": 3.2281882709722924, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.90180531819351e-06, "top_five_layers": [ 1, 22, 23, 24, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.112823883886449e-05, 3.971602382080164e-06, 1.0685844245017506e-06, 1.1686640846164664e-06, 9.83204358817602e-07, 1.3493386177287903e-06, 1.334196781499486e-06, 1.6468444528072723e-06, 1.0853799494725536e-06, 1.0981686955346959e-06, 1.0133109071830404e-06, 1.1350130080245435e-06, 9.764939932210837e-07, 1.3684866644325666e-06, 1.3214345244705328e-06, 1.5063559430927853e-06, 1.163652200375509e-06, 1.8332247009311686e-06, 1.8916454109785263e-06, 2.8027818643749924e-06, 4.834272203879664e-06, 8.918633284338284e-06, 8.456492651021108e-06, 7.16209115125821e-06, 5.137537300470285e-06, 4.688738954428118e-06, 3.74004434888775e-06, 3.3531034659972647e-06, 1.6689624544596882e-06, 1.535276396680274e-06, 1.4405162573893904e-06, 1.4522244100589887e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.3673139480706595e-05, "first_to_last_ratio": 2.257300425332895, "imbalance_abs_log_ratio": 0.8141695972328834, "last_quartile_mean": 1.0487367660516611e-05, "mean": 2.6041475749138954e-05, "normalized": [ 0.6771788519354571, 1.2979325532283608, 0.9303271057055151, 0.9639489120456851, 0.5271438854731791, 0.9092278846604233, 1.028243959121235, 0.9384386122687313, 0.6892881365249813, 0.5698927083153338, 0.44101071224628413, 0.3889922048793135, 0.37294517808508393, 0.6424365166247796, 0.5244231346818359, 0.4778644993700143, 1.0150499086102556, 1.1538261154181413, 1.4466435534537925, 1.6137729430394836, 2.6468294071482537, 4.220741079232897, 2.9670008977788065, 2.335098275492907, 0.997666272782542, 0.8091786594449493, 0.3691954363618722, 0.3097208222153953, 0.23815117281925108, 0.1573714206968098, 0.14298175278225358, 0.19747742755617637 ], "peak_layer": 22, "peak_normalized": 4.220741079232897, "population_cv": 0.8898631795325547, "rest_mean": 1.816408757804311e-05, "spike_contrast": 3.7755473033479423, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.857937187305651e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.7634736650506966e-05, 3.380007910891436e-05, 2.4227090761996806e-05, 2.5102652216446586e-05, 1.3727604709856678e-05, 2.3677635908825323e-05, 2.6776990125654265e-05, 2.443832636345178e-05, 1.7950080291484483e-05, 1.4840847143204883e-05, 1.1484569768072106e-05, 1.0129931069968734e-05, 9.712042810861021e-06, 1.6729994968045503e-05, 1.3656752344104461e-05, 1.2444296771718655e-05, 2.643339757923968e-05, 3.004733480338473e-05, 3.767273301491514e-05, 4.202502896077931e-05, 6.892734381835908e-05, 0.00010991432645823807, 7.726508192718029e-05, 6.080940511310473e-05, 2.5980702048400417e-05, 2.1072206436656415e-05, 9.614394002710469e-06, 8.065587280725595e-06, 6.201807991601527e-06, 4.098184035683516e-06, 3.723455847648438e-06, 5.142603640706511e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.5893040995251795e-05, "first_to_last_ratio": 2.4728739688838175, "imbalance_abs_log_ratio": 0.9053810243954281, "last_quartile_mean": 6.426951472349174e-06, "mean": 1.7339248245207273e-05, "normalized": [ 0.5512149265120582, 1.2985625489006958, 0.9783452688543264, 0.9971647574760376, 0.5962146785243507, 0.9290000522923118, 1.0861697535899135, 0.8960754411880637, 0.726825196328683, 0.6059411861934751, 0.5099587939140718, 0.44057324634523176, 0.4391978251197769, 0.7191858964012756, 0.5811495621714656, 0.4881322826391351, 1.0011626163335512, 1.1990369599246455, 1.4118886136322046, 1.554688514867236, 2.6196553178384634, 3.9391535171303675, 3.1321858798359785, 2.3332437535315793, 0.9448004540825558, 0.7676606562766751, 0.3665181536820326, 0.2919617681864151, 0.18930565608900826, 0.13559488549785165, 0.11350226730002422, 0.15592956934053803 ], "peak_layer": 22, "peak_normalized": 3.9391535171303675, "population_cv": 0.8670378924383861, "rest_mean": 1.2221576239594637e-05, "spike_contrast": 3.679940802546367, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.49746770755155e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.55765244725626e-06, 2.2516098397318274e-05, 1.6963771486189216e-05, 1.7290087271248922e-05, 1.0337914318370167e-05, 1.610816252650693e-05, 1.8833366993931122e-05, 1.5537274521193467e-05, 1.2602602510014549e-05, 1.0506564649404027e-05, 8.842302122502588e-06, 7.639208888576832e-06, 7.615360118506942e-06, 1.2470142792153638e-05, 1.0076696526084561e-05, 8.463846825179644e-06, 1.735940713842865e-05, 2.0790399503312074e-05, 2.4481087166350335e-05, 2.6957130103255622e-05, 4.542285387287848e-05, 6.830196070950478e-05, 5.430974852060899e-05, 4.0456692659063265e-05, 1.638212961552199e-05, 1.3310658687260002e-05, 6.355149253067793e-06, 5.062397576693911e-06, 3.282417765149148e-06, 2.351113380427705e-06, 1.968043989108992e-06, 2.7037015115638496e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.000305004112306e-05, "first_to_last_ratio": 2.193525307703265, "imbalance_abs_log_ratio": 0.7855099791473917, "last_quartile_mean": 3.191349094322504e-05, "mean": 7.690838336316119e-05, "normalized": [ 0.8923720562068232, 1.3111973164248405, 0.8460717362092315, 0.9445716926262052, 0.4741848629196519, 0.8656844065975011, 0.9924506003335006, 0.9551754577499273, 0.6234358560310297, 0.5152936103919111, 0.3846693562402596, 0.34526537659024253, 0.30979982474526946, 0.5791064055325631, 0.47493901082375733, 0.4350256040989574, 1.011874248892453, 1.1362510856108166, 1.416617477431854, 1.6892273518644711, 2.692715672842754, 4.577499824936365, 2.8907687343273323, 2.31616526773872, 1.0250323630610032, 0.8340050859269844, 0.36589411026523266, 0.29901912364743294, 0.2532838824218693, 0.16506308776319403, 0.15276161475417316, 0.22457789499367237 ], "peak_layer": 22, "peak_normalized": 4.577499824936365, "population_cv": 0.9353823752601559, "rest_mean": 5.269026995072771e-05, "spike_contrast": 3.9416422801499356, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000207686195790302, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 6.863089220132679e-05, 0.00010084206587634981, 6.507000944111496e-05, 7.264548185048625e-05, 3.646879122243263e-05, 6.657838821411133e-05, 7.632777123944834e-05, 7.346100028371438e-05, 4.7947443817975e-05, 3.9630398532608524e-05, 2.9584298317786306e-05, 2.6553801944828592e-05, 2.3826203687349334e-05, 4.453813744476065e-05, 3.652679151855409e-05, 3.34571159328334e-05, 7.782161264913157e-05, 8.738723408896476e-05, 0.00010894976003328338, 0.00012991574476473033, 0.00020709240925498307, 0.0003520481113810092, 0.00022232435003388673, 0.00017813252634368837, 7.883358193794265e-05, 6.414198287529871e-05, 2.8140324502601288e-05, 2.2997077394393273e-05, 1.9479653929010965e-05, 1.2694735232798848e-05, 1.174864883068949e-05, 1.7271922843065113e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.047033343464136124, 0.060984931886196136, 0.09079429507255554, 0.08531255275011063, 0.0786692351102829, 0.08076046407222748, 0.08904207497835159, 0.08403915911912918, 0.06012389436364174, 0.06841155141592026, 0.07131828367710114, 0.07358616590499878, 0.06597626209259033, 0.07235056161880493, 0.07261744141578674, 0.07352335751056671, 0.07520031183958054, 0.07378771901130676, 0.07715436071157455, 0.08649503439664841, 0.04556623473763466, 0.04546353965997696, 0.05167657509446144, 0.058827146887779236, 0.04798740893602371, 0.046526361256837845, 0.051699329167604446, 0.059578344225883484, 0.05090610682964325, 0.05821637064218521, 0.06188955530524254, 0.06796324998140335 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.364372159670893e-06, "first_to_last_ratio": 0.2681115118992595, "imbalance_abs_log_ratio": 1.3163522958360352, "last_quartile_mean": 1.627819756322424e-05, "mean": 1.0259845446114468e-05, "normalized": [ 1.0134993414800566, 0.522056581911947, 0.2644061262673726, 0.3833634649429024, 0.2440106441209935, 0.2461172286895863, 0.27872130983193855, 0.4508957935090263, 0.4833194134691065, 0.39025737906993885, 0.3708598519154414, 0.36981229946243677, 0.42398498232424564, 0.41928891386718925, 0.4397458055599365, 0.45615006609958, 0.4114730365687318, 0.5108641768312265, 0.48188059954330154, 0.7614115358485232, 2.335493650100753, 2.8449380275824225, 2.7201337953566025, 2.484573069357449, 2.4148997615580035, 2.4501695400243877, 2.1889991494613, 1.6914475251239836, 1.1580207584918483, 1.0537166329047938, 0.8749014303613178, 0.8605881083636573 ], "peak_layer": 22, "peak_normalized": 2.8449380275824225, "population_cv": 0.8504187150143712, "rest_mean": 7.295875539731006e-06, "spike_contrast": 3.6000179550148346, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6265282940585166e-05, "top_five_layers": [ 22, 23, 24, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.0398346603324171e-05, 5.356219844543375e-06, 2.7127659905090695e-06, 3.933249900001101e-06, 2.503511495888233e-06, 2.525124727981165e-06, 2.8596375614142744e-06, 4.626121153705753e-06, 4.9587824832997285e-06, 4.00398039346328e-06, 3.804964762821328e-06, 3.7942170365568018e-06, 4.350020390120335e-06, 4.301839453546563e-06, 4.511724000622053e-06, 4.680029178416589e-06, 4.221649760438595e-06, 5.241387498244876e-06, 4.944020474795252e-06, 7.811964678694494e-06, 2.3961803890415467e-05, 2.9188624466769397e-05, 2.7908152333111502e-05, 2.5491335691185668e-05, 2.4776498321443796e-05, 2.5138360797427595e-05, 2.2458792955148965e-05, 1.735399018798489e-05, 1.1881114005518612e-05, 1.081096979760332e-05, 8.976353456091601e-06, 8.829500984575134e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.76496694166417e-05, "first_to_last_ratio": 0.27059749062388033, "imbalance_abs_log_ratio": 1.3071228366665688, "last_quartile_mean": 6.522480816784082e-05, "mean": 4.117927500146834e-05, "normalized": [ 1.007463472142432, 0.5209651751221482, 0.2643970395144652, 0.3829104157055737, 0.2536844256534051, 0.25437837481906395, 0.28207653195017285, 0.46296966564632774, 0.49565891190391403, 0.39385639261196637, 0.3771016062127897, 0.37530908825337306, 0.4278590182802805, 0.42183720924091433, 0.44371630887119135, 0.4598267075919064, 0.41216967133849985, 0.5079852336529339, 0.47800548121281095, 0.7481511452590028, 2.3310574965471607, 2.8352891763903005, 2.718777452615949, 2.4731685677354274, 2.4074108105775847, 2.4550622811024825, 2.180382977806297, 1.6846495469817935, 1.161664542480338, 1.0519342613419045, 0.8725636415138176, 0.8577173699237725 ], "peak_layer": 22, "peak_normalized": 2.8352891763903005, "population_cv": 0.8467657487851442, "rest_mean": 2.9335347587928278e-05, "spike_contrast": 3.5839521832647088, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010513648303458467, "top_five_layers": [ 22, 23, 24, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.148661537328735e-05, 2.1452968212543055e-05, 1.0887678399740253e-05, 1.576797330926638e-05, 1.0446540727571119e-05, 1.0475117051100824e-05, 1.1615707080636639e-05, 1.906475517898798e-05, 2.0410874640219845e-05, 1.6218720702454448e-05, 1.552877074573189e-05, 1.5454956155736e-05, 1.761892417562194e-05, 1.7370950445183553e-05, 1.8271915905643255e-05, 1.8935330444946885e-05, 1.697284824331291e-05, 2.0918463633279316e-05, 1.968391916307155e-05, 3.080832175328396e-05, 9.599125769454986e-05, 0.00011675515270326287, 0.00011195728438906372, 0.00010184328857576475, 9.913543181028217e-05, 0.0001010976848192513, 8.978659025160596e-05, 6.937264697626233e-05, 4.783650365425274e-05, 4.3317890231264755e-05, 3.593153815018013e-05, 3.5320179449627176e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015651903115212917, "first_to_last_ratio": 0.2705974872956941, "imbalance_abs_log_ratio": 1.307122848965967, "last_quartile_mean": 0.05784201202914119, "mean": 0.03651819290826097, "normalized": [ 1.0074634186476772, 0.5209651919181703, 0.2643970221469926, 0.382910419856966, 0.2536844208302265, 0.25437835679028203, 0.2820765262777231, 0.4629696917045884, 0.4956588954855505, 0.393856398901423, 0.3771015816519749, 0.37530907952450476, 0.42785903141736564, 0.4218372178649866, 0.4437163224543097, 0.4598267050189124, 0.41216967747329053, 0.5079852485753572, 0.4780054808986325, 0.7481511542287224, 2.331057435590359, 2.835289207255424, 2.7187775380829278, 2.4731685834004287, 2.4074106915648685, 2.455062286976735, 2.180383000719517, 1.6846496365458605, 1.1616645069657927, 1.0519342829703067, 0.8725635941676015, 0.857717394092523 ], "peak_layer": 22, "peak_normalized": 2.835289207255424, "population_cv": 0.8467657515631118, "rest_mean": 0.026014879690828146, "spike_contrast": 3.5839521608577614, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09323608428239823, "top_five_layers": [ 22, 23, 24, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.036790743470191956, 0.01902470737695694, 0.009655301459133625, 0.013983196578919888, 0.00926409661769867, 0.009289437904953957, 0.010300925001502037, 0.016906816512346268, 0.018100567162036896, 0.01438292395323515, 0.013771068304777145, 0.013705609366297722, 0.01562463864684105, 0.015404732897877693, 0.01620371825993061, 0.016792040318250656, 0.01505169179290533, 0.018550703302025795, 0.017455896362662315, 0.02732112817466259, 0.08512600511312485, 0.10353963822126389, 0.09928484261035919, 0.09031564742326736, 0.08791428804397583, 0.08965443819761276, 0.07962364703416824, 0.06152036041021347, 0.042421888560056686, 0.038414739072322845, 0.03186444565653801, 0.03132228925824165 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.036145782506992e-06, "first_to_last_ratio": 0.23813875303874632, "imbalance_abs_log_ratio": 1.434901779225667, "last_quartile_mean": 4.35101708262664e-06, "mean": 2.601542721691885e-06, "normalized": [ 1.0445105925849996, 0.4868956651441851, 0.2653781288407282, 0.38704215458190727, 0.2311435484604591, 0.22762453648926426, 0.22559249684218238, 0.3180631703688383, 0.41371995096278646, 0.35456417415204133, 0.35816018513093856, 0.3384166475711158, 0.38409460459920464, 0.37901778296480176, 0.4237575276229331, 0.4302249635723777, 0.38747010650156044, 0.47861136725136255, 0.44057460034262785, 0.49825323727916515, 2.2524328318137035, 2.8621193651579846, 2.8149870576895797, 2.617539402453915, 2.4626516595822796, 2.6269385253668815, 2.362864616381361, 1.8136241394740558, 1.1783341687752071, 1.0893413201795874, 0.9454878584900481, 0.9005636133719171 ], "peak_layer": 22, "peak_normalized": 2.8621193651579846, "population_cv": 0.9010662730896349, "rest_mean": 1.8297776991689548e-06, "spike_contrast": 3.6993968427913813, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.76907384331571e-06, "top_five_layers": [ 22, 23, 26, 24, 25 ], "top_five_spike_overlap": 4 }, "values": [ 2.717338929869584e-06, 1.266679873879184e-06, 6.903925395818078e-07, 1.0069067002405063e-06, 6.013298161633429e-07, 5.921749561821343e-07, 5.868885182280792e-07, 8.274549259112973e-07, 1.0763101272459608e-06, 9.224138466379372e-07, 9.317690228272113e-07, 8.80405366388004e-07, 9.992385230361833e-07, 9.860309546638746e-07, 1.1024233117495896e-06, 1.1192486226718756e-06, 1.0080200354423141e-06, 1.245127918991784e-06, 1.1461736448836746e-06, 1.296227083003032e-06, 5.859800239704782e-06, 7.445925803040154e-06, 7.323309091589181e-06, 6.809640581195708e-06, 6.406693501048721e-06, 6.8340928010002244e-06, 6.147093245090218e-06, 4.718220679933438e-06, 3.0654866804979974e-06, 2.8339679829514353e-06, 2.4597270567028318e-06, 2.342854713788256e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2745702917982271e-05, "first_to_last_ratio": 0.3040242464136832, "imbalance_abs_log_ratio": 1.1906478228166633, "last_quartile_mean": 4.1923310618585674e-05, "mean": 2.8222844818515114e-05, "normalized": [ 1.092738547299169, 0.5627549631144219, 0.24809914749976855, 0.3821239330736069, 0.2717732575074452, 0.2708922129031054, 0.29822547774823177, 0.4862679168736732, 0.544040724965591, 0.43215299071277247, 0.4170763968939518, 0.42015115877685144, 0.47871065526968887, 0.46826286484802104, 0.4900402959326792, 0.5070735564042542, 0.4633460623422733, 0.5693312811282049, 0.5194930950029935, 0.7472397737749802, 2.399371006579266, 2.8986292719666995, 2.739777767666698, 2.408916709158605, 2.333408286273547, 2.300446003604439, 2.062640270367703, 1.6162771645238014, 1.0435838635901262, 0.9412033869198291, 0.8018738201820883, 0.7840781370955142 ], "peak_layer": 22, "peak_normalized": 2.8986292719666995, "population_cv": 0.8223755795225939, "rest_mean": 2.0090376640129965e-05, "spike_contrast": 3.5906829560228823, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.213817298179492e-05, "top_five_layers": [ 22, 23, 24, 21, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.0840190447634086e-05, 1.5882545994827524e-05, 7.0020637394918595e-06, 1.0784624464577064e-05, 7.670214472454973e-06, 7.6453488873085e-06, 8.416771379415877e-06, 1.3723863958148286e-05, 1.5354376955656335e-05, 1.219658679474378e-05, 1.1771082427003421e-05, 1.1857860954478383e-05, 1.3510576536646113e-05, 1.3215710168879014e-05, 1.3830331226927228e-05, 1.4311058293969836e-05, 1.3076944014756009e-05, 1.606814839760773e-05, 1.4661573004559614e-05, 2.1089232177473605e-05, 6.771707558073103e-05, 8.18075641291216e-05, 7.732432277407497e-05, 6.79864824633114e-05, 6.58554199617356e-05, 6.492513057310134e-05, 5.821357626700774e-05, 4.561593959806487e-05, 2.9452905437210575e-05, 2.6563437131699175e-05, 2.263116039102897e-05, 2.2128915588837117e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.982105043742195e-06, "first_to_last_ratio": 0.3255748540372994, "imbalance_abs_log_ratio": 1.122162877451286, "last_quartile_mean": 2.758844834716001e-05, "mean": 1.9313689406885715e-05, "normalized": [ 1.153292392223287, 0.5797609618245384, 0.23076786066825913, 0.37581241874572735, 0.2819306093436648, 0.2851688939012761, 0.30549556090944335, 0.5082847079850745, 0.583859969769007, 0.45907810601554777, 0.45116352890905354, 0.46235698273638653, 0.522619204122844, 0.5090842134923546, 0.5336639455934807, 0.5440001607440246, 0.5095787126873097, 0.6267236930541729, 0.5514908746704607, 0.7356132069066491, 2.430614280460028, 2.906404914690865, 2.7136484598097654, 2.3120655741144565, 2.2482332201724016, 2.1318721582988647, 1.9232163840966459, 1.5686764250532175, 1.0367548344938609, 0.9157520446280638, 0.8238220024989565, 0.7791936973803123 ], "peak_layer": 22, "peak_normalized": 2.906404914690865, "population_cv": 0.7900897792080465, "rest_mean": 1.3869398959219257e-05, "spike_contrast": 3.5122544219484153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.8712857824284585e-05, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.2274331058724783e-05, 1.119732314691646e-05, 4.456978786038235e-06, 7.258324330905452e-06, 5.4451202231575735e-06, 5.5076634453143924e-06, 5.900246378587326e-06, 9.816852980293334e-06, 1.1276490113232285e-05, 8.866491953085642e-06, 8.713632269063964e-06, 8.92981915967539e-06, 1.0093704986502416e-05, 9.832294381340034e-06, 1.0307019692845643e-05, 1.0506650141905993e-05, 9.841844985203352e-06, 1.2104346751584671e-05, 1.0651323464117013e-05, 1.4207405001798179e-05, 4.694412928074598e-05, 5.613340181298554e-05, 5.2410563512239605e-05, 4.4654516386799514e-05, 4.3421678128652275e-05, 4.117431672057137e-05, 3.714440390467644e-05, 3.0296929253381677e-05, 2.0023560864501633e-05, 1.768655056366697e-05, 1.5911042282823473e-05, 1.5049105059006251e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.531509560161794e-05, "first_to_last_ratio": 0.2551422661109217, "imbalance_abs_log_ratio": 1.36593398307824, "last_quartile_mean": 0.00013841334930475568, "mean": 8.573219730578785e-05, "normalized": [ 0.9822788357613834, 0.5147377658277125, 0.24077984042300446, 0.3633612864660074, 0.2505573845679934, 0.23871348492801034, 0.2720502738875204, 0.4329080853308244, 0.4899097166580031, 0.3804980486342757, 0.36593190899850525, 0.3643305688887499, 0.412819132135824, 0.40296081445560644, 0.4305724210670739, 0.4502160271371867, 0.3962072388182212, 0.4765863055500514, 0.454434200643566, 0.7197132038245179, 2.2718289409811727, 2.8618271883484865, 2.7794936015049534, 2.53140346971279, 2.4848609397336863, 2.575834812444881, 2.3585524326140215, 1.8181964500308416, 1.0732112926751614, 0.9812549829334379, 0.8189418899610229, 0.8050274550555072 ], "peak_layer": 22, "peak_normalized": 2.8618271883484865, "population_cv": 0.8817284503784036, "rest_mean": 6.055419369119961e-05, "spike_contrast": 3.6610745401896616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022169341682456433, "top_five_layers": [ 22, 23, 26, 24, 25 ], "top_five_spike_overlap": 4 }, "values": [ 8.42129229567945e-05, 4.4129599700681865e-05, 2.064258478640113e-05, 3.115176150458865e-05, 2.1480835130205378e-05, 2.0465431589400396e-05, 2.3323467758018523e-05, 3.7114161386853084e-05, 4.200103649054654e-05, 3.262093377998099e-05, 3.137214662274346e-05, 3.123486021650024e-05, 3.539189128787257e-05, 3.454671605140902e-05, 3.691391975735314e-05, 3.859800926875323e-05, 3.3967717172345147e-05, 4.08587911806535e-05, 3.895964255207218e-05, 6.170259439386427e-05, 0.00019476888701319695, 0.00024535073316656053, 0.00023829209385439754, 0.0002170227817259729, 0.0002130325883626938, 0.00022083197836764157, 0.0002022038825089112, 0.00015587797679472715, 9.200876229442656e-05, 8.412514580413699e-05, 7.020968769211322e-05, 6.9016772613395e-05 ] } } } } } }, "step": 2000 }, { "modes": { "learned": { "bits_per_byte": 1.9433676664104056, "forward": { "activation_sha256": { "attention_branch_output": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", "mlp_branch_output": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", "post_attention_state": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", "post_mlp_state": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", "pre_attention_input": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", "pre_mlp_input": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c" }, "logits_sha256": "000b42d2e473ee11592a1ced2fe607cfcfbbe21b9cb8637453b9d2bc3414fcfe", "loss_nats": 1.347039818763733, "mixer_summary_sha256": "7935548f1014624ed4c9fa535e6d7a55432ce9be32776c3985f675572d6d61e8" }, "loss_nats": 1.347039818763733, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.24809812009334564, "group": 1, "latest_source_mass": 0.07119978964328766, "layer": 1, "max_source_mass": 0.9288002252578735, "mixer_index": 1, "normalized_entropy": 0.35792992751254427, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9288002252578735, "median": 0.9386287331581116, "p05": 0.858244001865387, "p95": 0.9685006141662598 }, { "label": "current_group_partial", "mean": 0.07119978964328766, "median": 0.06137128174304962, "p05": 0.031499382108449936, "p95": 0.14175599813461304 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6067080497741699, "group": 1, "latest_source_mass": 0.43957531452178955, "layer": 2, "max_source_mass": 0.5604246854782104, "mixer_index": 2, "normalized_entropy": 0.8752946946766094, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5604246854782104, "median": 0.5413482189178467, "p05": 0.26798251271247864, "p95": 0.8787311315536499 }, { "label": "current_group_partial", "mean": 0.43957531452178955, "median": 0.4586517810821533, "p05": 0.1212688684463501, "p95": 0.7320175170898438 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3847596049308777, "group": 1, "latest_source_mass": 0.13518528640270233, "layer": 2, "max_source_mass": 0.8648147583007812, "mixer_index": 3, "normalized_entropy": 0.555090773968174, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8648147583007812, "median": 0.8810019493103027, "p05": 0.7563575506210327, "p95": 0.9309777617454529 }, { "label": "current_group_partial", "mean": 0.13518528640270233, "median": 0.11899800598621368, "p05": 0.06902224570512772, "p95": 0.24364244937896729 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6843481659889221, "group": 1, "latest_source_mass": 0.5056107044219971, "layer": 3, "max_source_mass": 0.5056107044219971, "mixer_index": 4, "normalized_entropy": 0.9873057053136751, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.49438929557800293, "median": 0.4973145127296448, "p05": 0.3897787928581238, "p95": 0.6022077798843384 }, { "label": "current_group_partial", "mean": 0.5056107044219971, "median": 0.502685546875, "p05": 0.3977922201156616, "p95": 0.6102211475372314 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5728601813316345, "group": 1, "latest_source_mass": 0.30453476309776306, "layer": 3, "max_source_mass": 0.6954652070999146, "mixer_index": 5, "normalized_entropy": 0.8264625427299015, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6954652070999146, "median": 0.7084143161773682, "p05": 0.453019917011261, "p95": 0.9048184752464294 }, { "label": "current_group_partial", "mean": 0.30453476309776306, "median": 0.29158568382263184, "p05": 0.09518152475357056, "p95": 0.5469801425933838 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.686518132686615, "group": 1, "latest_source_mass": 0.4736926555633545, "layer": 4, "max_source_mass": 0.5263072848320007, "mixer_index": 6, "normalized_entropy": 0.9904363055073309, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5263072848320007, "median": 0.5291417241096497, "p05": 0.4370424747467041, "p95": 0.6044135093688965 }, { "label": "current_group_partial", "mean": 0.4736926555633545, "median": 0.47085827589035034, "p05": 0.3955864906311035, "p95": 0.5629575252532959 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5044870376586914, "group": 1, "latest_source_mass": 0.2221563160419464, "layer": 4, "max_source_mass": 0.777843713760376, "mixer_index": 7, "normalized_entropy": 0.7278209474229579, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.777843713760376, "median": 0.7819972634315491, "p05": 0.6222297549247742, "p95": 0.9289263486862183 }, { "label": "current_group_partial", "mean": 0.2221563160419464, "median": 0.21800270676612854, "p05": 0.07107366621494293, "p95": 0.3777702748775482 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6724032163619995, "group": 2, "latest_source_mass": 0.4305936098098755, "layer": 5, "max_source_mass": 0.5694063901901245, "mixer_index": 8, "normalized_entropy": 0.9700727857232454, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5694063901901245, "median": 0.5728721022605896, "p05": 0.45058131217956543, "p95": 0.6837627291679382 }, { "label": "completed_group_1", "mean": 0.4305936098098755, "median": 0.4271278977394104, "p05": 0.31623730063438416, "p95": 0.5494186878204346 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9718273282051086, "group": 2, "latest_source_mass": 0.23954525589942932, "layer": 5, "max_source_mass": 0.5544795989990234, "mixer_index": 9, "normalized_entropy": 0.8845953556402437, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5544795989990234, "median": 0.5475102663040161, "p05": 0.40941980481147766, "p95": 0.7249865531921387 }, { "label": "completed_group_1", "mean": 0.20597517490386963, "median": 0.20141024887561798, "p05": 0.11906205117702484, "p95": 0.3046763241291046 }, { "label": "current_group_partial", "mean": 0.23954525589942932, "median": 0.2397485226392746, "p05": 0.14606745541095734, "p95": 0.33606359362602234 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0856361389160156, "group": 2, "latest_source_mass": 0.36206769943237305, "layer": 6, "max_source_mass": 0.3703838884830475, "mixer_index": 10, "normalized_entropy": 0.9881885994850598, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3703838884830475, "median": 0.3719092607498169, "p05": 0.31337594985961914, "p95": 0.42367395758628845 }, { "label": "completed_group_1", "mean": 0.26754841208457947, "median": 0.2691181004047394, "p05": 0.24584907293319702, "p95": 0.28693482279777527 }, { "label": "current_group_partial", "mean": 0.36206769943237305, "median": 0.3628925681114197, "p05": 0.31465041637420654, "p95": 0.4040696620941162 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.022249460220337, "group": 2, "latest_source_mass": 0.32249319553375244, "layer": 6, "max_source_mass": 0.46441033482551575, "mixer_index": 11, "normalized_entropy": 0.9304915580906614, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.46441033482551575, "median": 0.47390004992485046, "p05": 0.25647592544555664, "p95": 0.6490365266799927 }, { "label": "completed_group_1", "mean": 0.21309645473957062, "median": 0.2148953229188919, "p05": 0.13959459960460663, "p95": 0.27812671661376953 }, { "label": "current_group_partial", "mean": 0.32249319553375244, "median": 0.31052517890930176, "p05": 0.20389017462730408, "p95": 0.47258833050727844 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0742855072021484, "group": 2, "latest_source_mass": 0.36498188972473145, "layer": 7, "max_source_mass": 0.38770636916160583, "mixer_index": 12, "normalized_entropy": 0.9778568092521033, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.38770636916160583, "median": 0.4003474712371826, "p05": 0.2879825532436371, "p95": 0.45421719551086426 }, { "label": "completed_group_1", "mean": 0.24731171131134033, "median": 0.24118143320083618, "p05": 0.21154659986495972, "p95": 0.2994430363178253 }, { "label": "current_group_partial", "mean": 0.36498188972473145, "median": 0.35881954431533813, "p05": 0.31933724880218506, "p95": 0.42228373885154724 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0207322835922241, "group": 2, "latest_source_mass": 0.4325985312461853, "layer": 7, "max_source_mass": 0.4325985312461853, "mixer_index": 13, "normalized_entropy": 0.9291105644100317, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3404029607772827, "median": 0.2972515821456909, "p05": 0.1890696883201599, "p95": 0.675995409488678 }, { "label": "completed_group_1", "mean": 0.22699852287769318, "median": 0.2352103292942047, "p05": 0.11332616209983826, "p95": 0.2928347587585449 }, { "label": "current_group_partial", "mean": 0.4325985312461853, "median": 0.4621005654335022, "p05": 0.2054932415485382, "p95": 0.5461148023605347 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0628657341003418, "group": 2, "latest_source_mass": 0.361979603767395, "layer": 8, "max_source_mass": 0.41766518354415894, "mixer_index": 14, "normalized_entropy": 0.9674620838156608, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.41766518354415894, "median": 0.4145643711090088, "p05": 0.36852169036865234, "p95": 0.47784724831581116 }, { "label": "completed_group_1", "mean": 0.22035519778728485, "median": 0.2179521918296814, "p05": 0.191657155752182, "p95": 0.25349321961402893 }, { "label": "current_group_partial", "mean": 0.361979603767395, "median": 0.363964319229126, "p05": 0.3155593276023865, "p95": 0.39730241894721985 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.029329538345337, "group": 2, "latest_source_mass": 0.3858758807182312, "layer": 8, "max_source_mass": 0.42153143882751465, "mixer_index": 15, "normalized_entropy": 0.936936122927619, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.42153143882751465, "median": 0.41038113832473755, "p05": 0.29627615213394165, "p95": 0.6068807244300842 }, { "label": "completed_group_1", "mean": 0.19259268045425415, "median": 0.18828678131103516, "p05": 0.12331291288137436, "p95": 0.2707396447658539 }, { "label": "current_group_partial", "mean": 0.3858758807182312, "median": 0.39191579818725586, "p05": 0.2700805068016052, "p95": 0.4676054120063782 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0849454402923584, "group": 3, "latest_source_mass": 0.3364688754081726, "layer": 9, "max_source_mass": 0.37970075011253357, "mixer_index": 16, "normalized_entropy": 0.9875598985040298, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.37970075011253357, "median": 0.382883220911026, "p05": 0.29113224148750305, "p95": 0.46186232566833496 }, { "label": "completed_group_1", "mean": 0.2838304042816162, "median": 0.282575786113739, "p05": 0.25098085403442383, "p95": 0.31989675760269165 }, { "label": "completed_group_2", "mean": 0.3364688754081726, "median": 0.33596307039260864, "p05": 0.28113189339637756, "p95": 0.3942520022392273 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3144235610961914, "group": 3, "latest_source_mass": 0.30927807092666626, "layer": 9, "max_source_mass": 0.3205832242965698, "mixer_index": 17, "normalized_entropy": 0.9481561766105434, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3205832242965698, "median": 0.3228584825992584, "p05": 0.20318874716758728, "p95": 0.49334216117858887 }, { "label": "completed_group_1", "mean": 0.13048580288887024, "median": 0.1293262243270874, "p05": 0.08803658187389374, "p95": 0.17583128809928894 }, { "label": "completed_group_2", "mean": 0.2396528720855713, "median": 0.23850613832473755, "p05": 0.15605276823043823, "p95": 0.3097888231277466 }, { "label": "current_group_partial", "mean": 0.30927807092666626, "median": 0.3110916018486023, "p05": 0.24646314978599548, "p95": 0.36450129747390747 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3647149801254272, "group": 3, "latest_source_mass": 0.24869562685489655, "layer": 10, "max_source_mass": 0.3188267946243286, "mixer_index": 18, "normalized_entropy": 0.9844337670269171, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3188267946243286, "median": 0.32193225622177124, "p05": 0.2530626356601715, "p95": 0.38645845651626587 }, { "label": "completed_group_1", "mean": 0.2379944920539856, "median": 0.23598769307136536, "p05": 0.21630848944187164, "p95": 0.2653774321079254 }, { "label": "completed_group_2", "mean": 0.19448311626911163, "median": 0.19229988753795624, "p05": 0.16299879550933838, "p95": 0.23252975940704346 }, { "label": "current_group_partial", "mean": 0.24869562685489655, "median": 0.24343957006931305, "p05": 0.21172824501991272, "p95": 0.29676955938339233 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2778496742248535, "group": 3, "latest_source_mass": 0.2590479850769043, "layer": 10, "max_source_mass": 0.37896889448165894, "mixer_index": 19, "normalized_entropy": 0.9217736940028869, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.37896889448165894, "median": 0.3914150297641754, "p05": 0.14146459102630615, "p95": 0.6288160681724548 }, { "label": "completed_group_1", "mean": 0.16353653371334076, "median": 0.162285715341568, "p05": 0.08837783336639404, "p95": 0.23340767621994019 }, { "label": "completed_group_2", "mean": 0.1984466314315796, "median": 0.19906753301620483, "p05": 0.09471487253904343, "p95": 0.29035234451293945 }, { "label": "current_group_partial", "mean": 0.2590479850769043, "median": 0.2469818890094757, "p05": 0.1562758982181549, "p95": 0.3916419744491577 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3567583560943604, "group": 3, "latest_source_mass": 0.3088001608848572, "layer": 11, "max_source_mass": 0.3088001608848572, "mixer_index": 20, "normalized_entropy": 0.978694276010998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29770222306251526, "median": 0.29406338930130005, "p05": 0.24821004271507263, "p95": 0.3566911518573761 }, { "label": "completed_group_1", "mean": 0.21492168307304382, "median": 0.21014970541000366, "p05": 0.18107125163078308, "p95": 0.26316314935684204 }, { "label": "completed_group_2", "mean": 0.17857596278190613, "median": 0.17864474654197693, "p05": 0.15527817606925964, "p95": 0.2019544541835785 }, { "label": "current_group_partial", "mean": 0.3088001608848572, "median": 0.309349000453949, "p05": 0.2659364938735962, "p95": 0.3488244414329529 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2773678302764893, "group": 3, "latest_source_mass": 0.24117937684059143, "layer": 11, "max_source_mass": 0.3856483995914459, "mixer_index": 21, "normalized_entropy": 0.9214261170654932, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3856483995914459, "median": 0.4236947298049927, "p05": 0.12469026446342468, "p95": 0.588103175163269 }, { "label": "completed_group_1", "mean": 0.18032220005989075, "median": 0.17527258396148682, "p05": 0.10318915545940399, "p95": 0.2606950104236603 }, { "label": "completed_group_2", "mean": 0.1928499937057495, "median": 0.18568618595600128, "p05": 0.12013933062553406, "p95": 0.28692901134490967 }, { "label": "current_group_partial", "mean": 0.24117937684059143, "median": 0.2192210555076599, "p05": 0.14662782847881317, "p95": 0.4063720405101776 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3655540943145752, "group": 3, "latest_source_mass": 0.29187142848968506, "layer": 12, "max_source_mass": 0.29819542169570923, "mixer_index": 22, "normalized_entropy": 0.9850390599666288, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29819542169570923, "median": 0.29867103695869446, "p05": 0.2453947365283966, "p95": 0.34859126806259155 }, { "label": "completed_group_1", "mean": 0.20063453912734985, "median": 0.19907723367214203, "p05": 0.16811034083366394, "p95": 0.23844538629055023 }, { "label": "completed_group_2", "mean": 0.20929867029190063, "median": 0.21011404693126678, "p05": 0.18459835648536682, "p95": 0.23381994664669037 }, { "label": "current_group_partial", "mean": 0.29187142848968506, "median": 0.29417213797569275, "p05": 0.24766536056995392, "p95": 0.32690057158470154 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2754871845245361, "group": 3, "latest_source_mass": 0.22957558929920197, "layer": 12, "max_source_mass": 0.42651012539863586, "mixer_index": 23, "normalized_entropy": 0.9200695179154873, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.42651012539863586, "median": 0.44201791286468506, "p05": 0.24115881323814392, "p95": 0.5910159945487976 }, { "label": "completed_group_1", "mean": 0.1713070571422577, "median": 0.170497864484787, "p05": 0.12993231415748596, "p95": 0.21631143987178802 }, { "label": "completed_group_2", "mean": 0.1726072132587433, "median": 0.16530022025108337, "p05": 0.11561842262744904, "p95": 0.2528795301914215 }, { "label": "current_group_partial", "mean": 0.22957558929920197, "median": 0.2182570993900299, "p05": 0.14037305116653442, "p95": 0.3425685167312622 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3596348762512207, "group": 4, "latest_source_mass": 0.3104133605957031, "layer": 13, "max_source_mass": 0.3104133605957031, "mixer_index": 24, "normalized_entropy": 0.9807692466936578, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2750997543334961, "median": 0.2747533321380615, "p05": 0.2329672873020172, "p95": 0.31678152084350586 }, { "label": "completed_group_1", "mean": 0.22746169567108154, "median": 0.215274840593338, "p05": 0.18571145832538605, "p95": 0.3119947016239166 }, { "label": "completed_group_2", "mean": 0.18702521920204163, "median": 0.1861300766468048, "p05": 0.16768434643745422, "p95": 0.20858913660049438 }, { "label": "completed_group_3", "mean": 0.3104133605957031, "median": 0.32007908821105957, "p05": 0.2161477506160736, "p95": 0.3817093074321747 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5752925872802734, "group": 4, "latest_source_mass": 0.20874352753162384, "layer": 13, "max_source_mass": 0.28211504220962524, "mixer_index": 25, "normalized_entropy": 0.9787843166300303, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.28211504220962524, "median": 0.2998196482658386, "p05": 0.19197332859039307, "p95": 0.3606031835079193 }, { "label": "completed_group_1", "mean": 0.1675817370414734, "median": 0.16741523146629333, "p05": 0.14403808116912842, "p95": 0.19205224514007568 }, { "label": "completed_group_2", "mean": 0.1759152114391327, "median": 0.17603915929794312, "p05": 0.138168603181839, "p95": 0.21206450462341309 }, { "label": "completed_group_3", "mean": 0.16564446687698364, "median": 0.16148409247398376, "p05": 0.12781348824501038, "p95": 0.21234022080898285 }, { "label": "current_group_partial", "mean": 0.20874352753162384, "median": 0.2069983333349228, "p05": 0.17109251022338867, "p95": 0.24858909845352173 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5704567432403564, "group": 4, "latest_source_mass": 0.29884350299835205, "layer": 14, "max_source_mass": 0.29884350299835205, "mixer_index": 26, "normalized_entropy": 0.975779637789948, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21000900864601135, "median": 0.21329987049102783, "p05": 0.17762595415115356, "p95": 0.2425374686717987 }, { "label": "completed_group_1", "mean": 0.18079037964344025, "median": 0.17888139188289642, "p05": 0.15963314473628998, "p95": 0.20616433024406433 }, { "label": "completed_group_2", "mean": 0.13544103503227234, "median": 0.1345825046300888, "p05": 0.12005693465471268, "p95": 0.1533820480108261 }, { "label": "completed_group_3", "mean": 0.1749160885810852, "median": 0.17272356152534485, "p05": 0.15442612767219543, "p95": 0.20184364914894104 }, { "label": "current_group_partial", "mean": 0.29884350299835205, "median": 0.3006042242050171, "p05": 0.2674207091331482, "p95": 0.3227018713951111 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.579707384109497, "group": 4, "latest_source_mass": 0.24509160220623016, "layer": 14, "max_source_mass": 0.24509160220623016, "mixer_index": 27, "normalized_entropy": 0.98152738412901, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21335941553115845, "median": 0.21055345237255096, "p05": 0.12470410019159317, "p95": 0.33470356464385986 }, { "label": "completed_group_1", "mean": 0.18788450956344604, "median": 0.18773096799850464, "p05": 0.14719606935977936, "p95": 0.22808396816253662 }, { "label": "completed_group_2", "mean": 0.20250459015369415, "median": 0.20350772142410278, "p05": 0.15922458469867706, "p95": 0.24294495582580566 }, { "label": "completed_group_3", "mean": 0.1511598825454712, "median": 0.14913730323314667, "p05": 0.12029191851615906, "p95": 0.1872422844171524 }, { "label": "current_group_partial", "mean": 0.24509160220623016, "median": 0.23680704832077026, "p05": 0.18374761939048767, "p95": 0.3342677652835846 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5663042068481445, "group": 4, "latest_source_mass": 0.29828590154647827, "layer": 15, "max_source_mass": 0.29828590154647827, "mixer_index": 28, "normalized_entropy": 0.9731995218624366, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21676908433437347, "median": 0.2142370641231537, "p05": 0.1767536848783493, "p95": 0.2722740173339844 }, { "label": "completed_group_1", "mean": 0.19467836618423462, "median": 0.19520726799964905, "p05": 0.16570945084095, "p95": 0.2210082858800888 }, { "label": "completed_group_2", "mean": 0.13713446259498596, "median": 0.13748224079608917, "p05": 0.12025479972362518, "p95": 0.15284715592861176 }, { "label": "completed_group_3", "mean": 0.15313217043876648, "median": 0.153159961104393, "p05": 0.13474825024604797, "p95": 0.1712232530117035 }, { "label": "current_group_partial", "mean": 0.29828590154647827, "median": 0.29950791597366333, "p05": 0.2589420676231384, "p95": 0.33298033475875854 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5384478569030762, "group": 4, "latest_source_mass": 0.29070425033569336, "layer": 15, "max_source_mass": 0.29070425033569336, "mixer_index": 29, "normalized_entropy": 0.9558913984922479, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21904908120632172, "median": 0.20143723487854004, "p05": 0.09117002040147781, "p95": 0.4469224512577057 }, { "label": "completed_group_1", "mean": 0.15864285826683044, "median": 0.15922614932060242, "p05": 0.10943489521741867, "p95": 0.2047940045595169 }, { "label": "completed_group_2", "mean": 0.18178996443748474, "median": 0.18768036365509033, "p05": 0.10846641659736633, "p95": 0.23086166381835938 }, { "label": "completed_group_3", "mean": 0.14981383085250854, "median": 0.14754682779312134, "p05": 0.10115746408700943, "p95": 0.19903366267681122 }, { "label": "current_group_partial", "mean": 0.29070425033569336, "median": 0.2898399233818054, "p05": 0.20691633224487305, "p95": 0.37558960914611816 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.552668571472168, "group": 4, "latest_source_mass": 0.30334532260894775, "layer": 16, "max_source_mass": 0.30334532260894775, "mixer_index": 30, "normalized_entropy": 0.9647272252484255, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22710487246513367, "median": 0.2245989590883255, "p05": 0.16175971925258636, "p95": 0.2988150715827942 }, { "label": "completed_group_1", "mean": 0.1922147125005722, "median": 0.1916719675064087, "p05": 0.15179888904094696, "p95": 0.23553937673568726 }, { "label": "completed_group_2", "mean": 0.13047492504119873, "median": 0.12986470758914948, "p05": 0.10791919380426407, "p95": 0.15449906885623932 }, { "label": "completed_group_3", "mean": 0.14686015248298645, "median": 0.14501844346523285, "p05": 0.12235601246356964, "p95": 0.17693395912647247 }, { "label": "current_group_partial", "mean": 0.30334532260894775, "median": 0.29679566621780396, "p05": 0.24902349710464478, "p95": 0.3774981200695038 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5210195779800415, "group": 4, "latest_source_mass": 0.282525897026062, "layer": 16, "max_source_mass": 0.282525897026062, "mixer_index": 31, "normalized_entropy": 0.9450625999481175, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2812899947166443, "median": 0.2745441794395447, "p05": 0.15230709314346313, "p95": 0.4724813997745514 }, { "label": "completed_group_1", "mean": 0.14511607587337494, "median": 0.14084157347679138, "p05": 0.10224629938602448, "p95": 0.2002197802066803 }, { "label": "completed_group_2", "mean": 0.15909329056739807, "median": 0.15865719318389893, "p05": 0.10898765176534653, "p95": 0.21094731986522675 }, { "label": "completed_group_3", "mean": 0.1319747418165207, "median": 0.1301102191209793, "p05": 0.08913854509592056, "p95": 0.17862913012504578 }, { "label": "current_group_partial", "mean": 0.282525897026062, "median": 0.28236913681030273, "p05": 0.19806379079818726, "p95": 0.36864838004112244 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5376527309417725, "group": 5, "latest_source_mass": 0.287123441696167, "layer": 17, "max_source_mass": 0.287123441696167, "mixer_index": 32, "normalized_entropy": 0.9553973589551147, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.23905543982982635, "median": 0.23701313138008118, "p05": 0.18620719015598297, "p95": 0.2925828695297241 }, { "label": "completed_group_1", "mean": 0.21673071384429932, "median": 0.2044925093650818, "p05": 0.15778198838233948, "p95": 0.321443647146225 }, { "label": "completed_group_2", "mean": 0.0950920432806015, "median": 0.09378060698509216, "p05": 0.07889896631240845, "p95": 0.11442621052265167 }, { "label": "completed_group_3", "mean": 0.16199833154678345, "median": 0.16117054224014282, "p05": 0.12811943888664246, "p95": 0.19840528070926666 }, { "label": "completed_group_4", "mean": 0.287123441696167, "median": 0.29111331701278687, "p05": 0.2107674777507782, "p95": 0.34516024589538574 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7209875583648682, "group": 5, "latest_source_mass": 0.1365262269973755, "layer": 17, "max_source_mass": 0.25328749418258667, "mixer_index": 33, "normalized_entropy": 0.9605014444859178, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1703755259513855, "median": 0.1390259861946106, "p05": 0.08010496944189072, "p95": 0.3672885000705719 }, { "label": "completed_group_1", "mean": 0.14151468873023987, "median": 0.1361694037914276, "p05": 0.10555511713027954, "p95": 0.19257718324661255 }, { "label": "completed_group_2", "mean": 0.17087554931640625, "median": 0.17344608902931213, "p05": 0.10519347339868546, "p95": 0.22031483054161072 }, { "label": "completed_group_3", "mean": 0.12742051482200623, "median": 0.12381769716739655, "p05": 0.0888485461473465, "p95": 0.17263755202293396 }, { "label": "completed_group_4", "mean": 0.25328749418258667, "median": 0.25078505277633667, "p05": 0.163196861743927, "p95": 0.35207411646842957 }, { "label": "current_group_partial", "mean": 0.1365262269973755, "median": 0.13269345462322235, "p05": 0.0852486714720726, "p95": 0.20064347982406616 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7200093269348145, "group": 5, "latest_source_mass": 0.27521812915802, "layer": 18, "max_source_mass": 0.27521812915802, "mixer_index": 34, "normalized_entropy": 0.9599554831295783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.18992580473423004, "median": 0.19383516907691956, "p05": 0.1371862292289734, "p95": 0.23474527895450592 }, { "label": "completed_group_1", "mean": 0.16036251187324524, "median": 0.1590055525302887, "p05": 0.14324960112571716, "p95": 0.1809857338666916 }, { "label": "completed_group_2", "mean": 0.08439569175243378, "median": 0.08315694332122803, "p05": 0.07186377048492432, "p95": 0.10051116347312927 }, { "label": "completed_group_3", "mean": 0.11250905692577362, "median": 0.11015664041042328, "p05": 0.09692119807004929, "p95": 0.13486211001873016 }, { "label": "completed_group_4", "mean": 0.1775888353586197, "median": 0.1758042573928833, "p05": 0.1488552987575531, "p95": 0.21211811900138855 }, { "label": "current_group_partial", "mean": 0.27521812915802, "median": 0.2752543091773987, "p05": 0.25022760033607483, "p95": 0.3000151515007019 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.753357172012329, "group": 5, "latest_source_mass": 0.24494445323944092, "layer": 18, "max_source_mass": 0.24494445323944092, "mixer_index": 35, "normalized_entropy": 0.978567269839924, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.15539543330669403, "median": 0.14567650854587555, "p05": 0.1150466799736023, "p95": 0.23550888895988464 }, { "label": "completed_group_1", "mean": 0.1388990879058838, "median": 0.1368958055973053, "p05": 0.11385980248451233, "p95": 0.17140795290470123 }, { "label": "completed_group_2", "mean": 0.15745997428894043, "median": 0.15778976678848267, "p05": 0.13043276965618134, "p95": 0.1834889054298401 }, { "label": "completed_group_3", "mean": 0.12427005916833878, "median": 0.12222100794315338, "p05": 0.10104347765445709, "p95": 0.1512138545513153 }, { "label": "completed_group_4", "mean": 0.17903099954128265, "median": 0.1781059056520462, "p05": 0.14208883047103882, "p95": 0.21939483284950256 }, { "label": "current_group_partial", "mean": 0.24494445323944092, "median": 0.24745827913284302, "p05": 0.16898170113563538, "p95": 0.3117744028568268 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7199265956878662, "group": 5, "latest_source_mass": 0.2730581760406494, "layer": 19, "max_source_mass": 0.2730581760406494, "mixer_index": 36, "normalized_entropy": 0.9599093099415087, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.19207912683486938, "median": 0.19417056441307068, "p05": 0.14205873012542725, "p95": 0.2466108500957489 }, { "label": "completed_group_1", "mean": 0.17904852330684662, "median": 0.17943383753299713, "p05": 0.15607136487960815, "p95": 0.2006712406873703 }, { "label": "completed_group_2", "mean": 0.08410340547561646, "median": 0.08364531397819519, "p05": 0.07357755303382874, "p95": 0.09571512043476105 }, { "label": "completed_group_3", "mean": 0.1121990978717804, "median": 0.11130240559577942, "p05": 0.0962604433298111, "p95": 0.13100717961788177 }, { "label": "completed_group_4", "mean": 0.15951167047023773, "median": 0.1591540426015854, "p05": 0.13084900379180908, "p95": 0.1905163675546646 }, { "label": "current_group_partial", "mean": 0.2730581760406494, "median": 0.27290621399879456, "p05": 0.241956889629364, "p95": 0.3052297830581665 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6756346225738525, "group": 5, "latest_source_mass": 0.3124367296695709, "layer": 19, "max_source_mass": 0.3124367296695709, "mixer_index": 37, "normalized_entropy": 0.9351894890756556, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15863081812858582, "median": 0.1342058777809143, "p05": 0.07548131048679352, "p95": 0.3739146590232849 }, { "label": "completed_group_1", "mean": 0.11242728680372238, "median": 0.11213450133800507, "p05": 0.08389876782894135, "p95": 0.14075449109077454 }, { "label": "completed_group_2", "mean": 0.15970337390899658, "median": 0.16262288391590118, "p05": 0.10122501850128174, "p95": 0.2050495743751526 }, { "label": "completed_group_3", "mean": 0.09774985909461975, "median": 0.09510388225317001, "p05": 0.06857988238334656, "p95": 0.12727102637290955 }, { "label": "completed_group_4", "mean": 0.15905189514160156, "median": 0.1553158164024353, "p05": 0.11221202462911606, "p95": 0.2187761515378952 }, { "label": "current_group_partial", "mean": 0.3124367296695709, "median": 0.3196638822555542, "p05": 0.18551334738731384, "p95": 0.4139381945133209 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7271867990493774, "group": 5, "latest_source_mass": 0.2614278793334961, "layer": 20, "max_source_mass": 0.2614278793334961, "mixer_index": 38, "normalized_entropy": 0.9639613065884912, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19560003280639648, "median": 0.1968384087085724, "p05": 0.1501731276512146, "p95": 0.2440420687198639 }, { "label": "completed_group_1", "mean": 0.18333327770233154, "median": 0.18228237330913544, "p05": 0.1655699908733368, "p95": 0.2043052464723587 }, { "label": "completed_group_2", "mean": 0.08475088328123093, "median": 0.08410285413265228, "p05": 0.07304102182388306, "p95": 0.09892263263463974 }, { "label": "completed_group_3", "mean": 0.11598999798297882, "median": 0.1141754686832428, "p05": 0.10139824450016022, "p95": 0.1362377256155014 }, { "label": "completed_group_4", "mean": 0.15889792144298553, "median": 0.15817126631736755, "p05": 0.13111311197280884, "p95": 0.18958830833435059 }, { "label": "current_group_partial", "mean": 0.2614278793334961, "median": 0.2622074484825134, "p05": 0.2309473156929016, "p95": 0.2893397808074951 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6916961669921875, "group": 5, "latest_source_mass": 0.3091394901275635, "layer": 20, "max_source_mass": 0.3091394901275635, "mixer_index": 39, "normalized_entropy": 0.9441536076943532, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19065655767917633, "median": 0.1910099983215332, "p05": 0.09996911883354187, "p95": 0.313973069190979 }, { "label": "completed_group_1", "mean": 0.12983909249305725, "median": 0.12771058082580566, "p05": 0.10737630724906921, "p95": 0.15882958471775055 }, { "label": "completed_group_2", "mean": 0.11857748031616211, "median": 0.11902420967817307, "p05": 0.08811743557453156, "p95": 0.14735448360443115 }, { "label": "completed_group_3", "mean": 0.09883742034435272, "median": 0.0951075479388237, "p05": 0.07493072748184204, "p95": 0.1301351934671402 }, { "label": "completed_group_4", "mean": 0.1529499888420105, "median": 0.15091747045516968, "p05": 0.11475001275539398, "p95": 0.19942505657672882 }, { "label": "current_group_partial", "mean": 0.3091394901275635, "median": 0.3146688640117645, "p05": 0.2252732366323471, "p95": 0.3756124973297119 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7504222393035889, "group": 6, "latest_source_mass": 0.2152354121208191, "layer": 21, "max_source_mass": 0.2152354121208191, "mixer_index": 40, "normalized_entropy": 0.9769292527069633, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20809361338615417, "median": 0.21471115946769714, "p05": 0.13097207248210907, "p95": 0.2735797166824341 }, { "label": "completed_group_1", "mean": 0.19103102385997772, "median": 0.19057458639144897, "p05": 0.16917173564434052, "p95": 0.2132101207971573 }, { "label": "completed_group_2", "mean": 0.10743936896324158, "median": 0.10638970136642456, "p05": 0.09155704826116562, "p95": 0.12698283791542053 }, { "label": "completed_group_3", "mean": 0.1194620281457901, "median": 0.11578401178121567, "p05": 0.10080886632204056, "p95": 0.14768129587173462 }, { "label": "completed_group_4", "mean": 0.15873853862285614, "median": 0.15696901082992554, "p05": 0.13252565264701843, "p95": 0.19312383234500885 }, { "label": "completed_group_5", "mean": 0.2152354121208191, "median": 0.21616290509700775, "p05": 0.18420100212097168, "p95": 0.24399688839912415 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5379457473754883, "group": 6, "latest_source_mass": 0.515034556388855, "layer": 21, "max_source_mass": 0.515034556388855, "mixer_index": 41, "normalized_entropy": 0.7903477702308708, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09571269154548645, "median": 0.08428268879652023, "p05": 0.04900683835148811, "p95": 0.1901935338973999 }, { "label": "completed_group_1", "mean": 0.07078731805086136, "median": 0.06967149674892426, "p05": 0.05498572438955307, "p95": 0.08918046206235886 }, { "label": "completed_group_2", "mean": 0.0824701264500618, "median": 0.08152244985103607, "p05": 0.06332080066204071, "p95": 0.10414593666791916 }, { "label": "completed_group_3", "mean": 0.060012347996234894, "median": 0.058421529829502106, "p05": 0.04666804522275925, "p95": 0.07636430114507675 }, { "label": "completed_group_4", "mean": 0.08145605027675629, "median": 0.0806138813495636, "p05": 0.061518602073192596, "p95": 0.10454171150922775 }, { "label": "completed_group_5", "mean": 0.09452696144580841, "median": 0.09397821128368378, "p05": 0.06939592957496643, "p95": 0.12165899574756622 }, { "label": "current_group_partial", "mean": 0.515034556388855, "median": 0.5195302963256836, "p05": 0.41491252183914185, "p95": 0.5988897681236267 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7319592237472534, "group": 6, "latest_source_mass": 0.3903183937072754, "layer": 22, "max_source_mass": 0.3903183937072754, "mixer_index": 42, "normalized_entropy": 0.8900509741357138, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13678354024887085, "median": 0.14219121634960175, "p05": 0.0800691545009613, "p95": 0.18056681752204895 }, { "label": "completed_group_1", "mean": 0.11740624904632568, "median": 0.11779546737670898, "p05": 0.09323577582836151, "p95": 0.13856223225593567 }, { "label": "completed_group_2", "mean": 0.06668868660926819, "median": 0.06673324108123779, "p05": 0.05682401731610298, "p95": 0.07633707672357559 }, { "label": "completed_group_3", "mean": 0.07360264658927917, "median": 0.07287117838859558, "p05": 0.06238633394241333, "p95": 0.0858106017112732 }, { "label": "completed_group_4", "mean": 0.09287553280591965, "median": 0.0919380784034729, "p05": 0.07644987106323242, "p95": 0.11259186267852783 }, { "label": "completed_group_5", "mean": 0.12232495099306107, "median": 0.12153243273496628, "p05": 0.10421743988990784, "p95": 0.14276890456676483 }, { "label": "current_group_partial", "mean": 0.3903183937072754, "median": 0.382642537355423, "p05": 0.32760486006736755, "p95": 0.4824257791042328 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4663677215576172, "group": 6, "latest_source_mass": 0.5484944581985474, "layer": 22, "max_source_mass": 0.5484944581985474, "mixer_index": 43, "normalized_entropy": 0.7535639414129677, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09485072642564774, "median": 0.07708277553319931, "p05": 0.041287682950496674, "p95": 0.2252470850944519 }, { "label": "completed_group_1", "mean": 0.07167200744152069, "median": 0.07091978192329407, "p05": 0.05350632965564728, "p95": 0.09384822845458984 }, { "label": "completed_group_2", "mean": 0.0841417908668518, "median": 0.08349095284938812, "p05": 0.060824211686849594, "p95": 0.11007672548294067 }, { "label": "completed_group_3", "mean": 0.05784624069929123, "median": 0.056044802069664, "p05": 0.04343996196985245, "p95": 0.07539206743240356 }, { "label": "completed_group_4", "mean": 0.07052354514598846, "median": 0.06955709308385849, "p05": 0.048022691160440445, "p95": 0.09658979624509811 }, { "label": "completed_group_5", "mean": 0.0724712610244751, "median": 0.07230593264102936, "p05": 0.05036160722374916, "p95": 0.09567554295063019 }, { "label": "current_group_partial", "mean": 0.5484944581985474, "median": 0.5530485510826111, "p05": 0.4294937252998352, "p95": 0.6537213325500488 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.749481201171875, "group": 6, "latest_source_mass": 0.37420469522476196, "layer": 23, "max_source_mass": 0.37420469522476196, "mixer_index": 44, "normalized_entropy": 0.8990554892892669, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14530612528324127, "median": 0.1497679054737091, "p05": 0.08296400308609009, "p95": 0.20158618688583374 }, { "label": "completed_group_1", "mean": 0.12230369448661804, "median": 0.12149038910865784, "p05": 0.10498286038637161, "p95": 0.14227813482284546 }, { "label": "completed_group_2", "mean": 0.0660901814699173, "median": 0.06578399986028671, "p05": 0.05611872673034668, "p95": 0.07736622542142868 }, { "label": "completed_group_3", "mean": 0.07630561292171478, "median": 0.07523077726364136, "p05": 0.06339914351701736, "p95": 0.09089048206806183 }, { "label": "completed_group_4", "mean": 0.09453228116035461, "median": 0.09349468350410461, "p05": 0.07686570286750793, "p95": 0.11586424708366394 }, { "label": "completed_group_5", "mean": 0.12125740945339203, "median": 0.12067118287086487, "p05": 0.10250546038150787, "p95": 0.14243635535240173 }, { "label": "current_group_partial", "mean": 0.37420469522476196, "median": 0.36897724866867065, "p05": 0.30640295147895813, "p95": 0.4604528248310089 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.472255825996399, "group": 6, "latest_source_mass": 0.5438836812973022, "layer": 23, "max_source_mass": 0.5438836812973022, "mixer_index": 45, "normalized_entropy": 0.7565898285237576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10688216984272003, "median": 0.0909440815448761, "p05": 0.04993019625544548, "p95": 0.22096660733222961 }, { "label": "completed_group_1", "mean": 0.07465942203998566, "median": 0.07390505075454712, "p05": 0.05582176893949509, "p95": 0.09690531343221664 }, { "label": "completed_group_2", "mean": 0.08327433466911316, "median": 0.08253603428602219, "p05": 0.05906296521425247, "p95": 0.10942579060792923 }, { "label": "completed_group_3", "mean": 0.054589346051216125, "median": 0.052618082612752914, "p05": 0.039012178778648376, "p95": 0.07247936725616455 }, { "label": "completed_group_4", "mean": 0.0637175589799881, "median": 0.062460802495479584, "p05": 0.04167243465781212, "p95": 0.09087979793548584 }, { "label": "completed_group_5", "mean": 0.07299347221851349, "median": 0.07232271879911423, "p05": 0.05079510435461998, "p95": 0.09688537567853928 }, { "label": "current_group_partial", "mean": 0.5438836812973022, "median": 0.5471436977386475, "p05": 0.42356419563293457, "p95": 0.6585544347763062 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7924723625183105, "group": 6, "latest_source_mass": 0.3345995247364044, "layer": 24, "max_source_mass": 0.3345995247364044, "mixer_index": 46, "normalized_entropy": 0.9211485758417507, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15522946417331696, "median": 0.16076582670211792, "p05": 0.0863914042711258, "p95": 0.2195497751235962 }, { "label": "completed_group_1", "mean": 0.13724806904792786, "median": 0.13687172532081604, "p05": 0.1200050488114357, "p95": 0.1560649424791336 }, { "label": "completed_group_2", "mean": 0.06889298558235168, "median": 0.06847138702869415, "p05": 0.058477431535720825, "p95": 0.08076588809490204 }, { "label": "completed_group_3", "mean": 0.08198929578065872, "median": 0.08025878667831421, "p05": 0.06727518886327744, "p95": 0.10007832199335098 }, { "label": "completed_group_4", "mean": 0.09816981852054596, "median": 0.09709671884775162, "p05": 0.07776231318712234, "p95": 0.12178758531808853 }, { "label": "completed_group_5", "mean": 0.123870849609375, "median": 0.12361469119787216, "p05": 0.1038728728890419, "p95": 0.14525073766708374 }, { "label": "current_group_partial", "mean": 0.3345995247364044, "median": 0.3317973017692566, "p05": 0.265953928232193, "p95": 0.412749707698822 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5755163431167603, "group": 6, "latest_source_mass": 0.48653730750083923, "layer": 24, "max_source_mass": 0.48653730750083923, "mixer_index": 47, "normalized_entropy": 0.8096552371041545, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12274101376533508, "median": 0.10600537061691284, "p05": 0.06405961513519287, "p95": 0.25983333587646484 }, { "label": "completed_group_1", "mean": 0.08900541067123413, "median": 0.08780360221862793, "p05": 0.06876012682914734, "p95": 0.11320354044437408 }, { "label": "completed_group_2", "mean": 0.08551642298698425, "median": 0.08451470732688904, "p05": 0.06253526359796524, "p95": 0.1114322617650032 }, { "label": "completed_group_3", "mean": 0.06285787373781204, "median": 0.061114855110645294, "p05": 0.045210886746644974, "p95": 0.08213447034358978 }, { "label": "completed_group_4", "mean": 0.07429315149784088, "median": 0.07252945005893707, "p05": 0.049904488027095795, "p95": 0.10469712316989899 }, { "label": "completed_group_5", "mean": 0.07904879748821259, "median": 0.07821422815322876, "p05": 0.05642634630203247, "p95": 0.10449491441249847 }, { "label": "current_group_partial", "mean": 0.48653730750083923, "median": 0.49373340606689453, "p05": 0.3549031913280487, "p95": 0.5985680222511292 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8390922546386719, "group": 7, "latest_source_mass": 0.29349833726882935, "layer": 25, "max_source_mass": 0.29349833726882935, "mixer_index": 48, "normalized_entropy": 0.945106461123861, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15884169936180115, "median": 0.15805873274803162, "p05": 0.09690988063812256, "p95": 0.2504432201385498 }, { "label": "completed_group_1", "mean": 0.14726480841636658, "median": 0.14679506421089172, "p05": 0.12764519453048706, "p95": 0.16903650760650635 }, { "label": "completed_group_2", "mean": 0.08053630590438843, "median": 0.08051620423793793, "p05": 0.06807594001293182, "p95": 0.09329371899366379 }, { "label": "completed_group_3", "mean": 0.0909707322716713, "median": 0.08902134001255035, "p05": 0.07499626278877258, "p95": 0.11060230433940887 }, { "label": "completed_group_4", "mean": 0.10738551616668701, "median": 0.10607586801052094, "p05": 0.08672098070383072, "p95": 0.1325560212135315 }, { "label": "completed_group_5", "mean": 0.12150255590677261, "median": 0.12205702066421509, "p05": 0.0993877649307251, "p95": 0.14230164885520935 }, { "label": "completed_group_6", "mean": 0.29349833726882935, "median": 0.29394760727882385, "p05": 0.22660650312900543, "p95": 0.3548165559768677 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6550344228744507, "group": 7, "latest_source_mass": 0.3589511513710022, "layer": 25, "max_source_mass": 0.3589511513710022, "mixer_index": 49, "normalized_entropy": 0.795903318127166, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0844201073050499, "median": 0.0718768984079361, "p05": 0.02839931659400463, "p95": 0.21728233993053436 }, { "label": "completed_group_1", "mean": 0.06695197522640228, "median": 0.06580687314271927, "p05": 0.04618899151682854, "p95": 0.09124172478914261 }, { "label": "completed_group_2", "mean": 0.061310961842536926, "median": 0.06096746027469635, "p05": 0.040395740419626236, "p95": 0.08348730951547623 }, { "label": "completed_group_3", "mean": 0.0447731651365757, "median": 0.042184896767139435, "p05": 0.02882835641503334, "p95": 0.06167382374405861 }, { "label": "completed_group_4", "mean": 0.04748315364122391, "median": 0.04516277089715004, "p05": 0.0278231929987669, "p95": 0.0743657797574997 }, { "label": "completed_group_5", "mean": 0.045562684535980225, "median": 0.044303566217422485, "p05": 0.03103901632130146, "p95": 0.06420586258172989 }, { "label": "completed_group_6", "mean": 0.2905467748641968, "median": 0.2895723581314087, "p05": 0.1524062305688858, "p95": 0.43639296293258667 }, { "label": "current_group_partial", "mean": 0.3589511513710022, "median": 0.3606260418891907, "p05": 0.16963356733322144, "p95": 0.546379804611206 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8679088354110718, "group": 7, "latest_source_mass": 0.32435470819473267, "layer": 26, "max_source_mass": 0.32435470819473267, "mixer_index": 50, "normalized_entropy": 0.8982742712267442, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11674155294895172, "median": 0.12052157521247864, "p05": 0.05360257253050804, "p95": 0.17403334379196167 }, { "label": "completed_group_1", "mean": 0.1088850274682045, "median": 0.1090676486492157, "p05": 0.08014046400785446, "p95": 0.13653376698493958 }, { "label": "completed_group_2", "mean": 0.05779466778039932, "median": 0.05755443871021271, "p05": 0.04364454001188278, "p95": 0.07319164276123047 }, { "label": "completed_group_3", "mean": 0.06273135542869568, "median": 0.06075543537735939, "p05": 0.04851231724023819, "p95": 0.07996035367250443 }, { "label": "completed_group_4", "mean": 0.07066236436367035, "median": 0.06902407109737396, "p05": 0.052817195653915405, "p95": 0.09365298599004745 }, { "label": "completed_group_5", "mean": 0.08129343390464783, "median": 0.08064926415681839, "p05": 0.06491829454898834, "p95": 0.09953147917985916 }, { "label": "completed_group_6", "mean": 0.17753681540489197, "median": 0.17456364631652832, "p05": 0.11964619904756546, "p95": 0.24453631043434143 }, { "label": "current_group_partial", "mean": 0.32435470819473267, "median": 0.32046452164649963, "p05": 0.19449321925640106, "p95": 0.4612511992454529 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5923268795013428, "group": 7, "latest_source_mass": 0.4404740333557129, "layer": 26, "max_source_mass": 0.4404740333557129, "mixer_index": 51, "normalized_entropy": 0.7657473641769285, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08656083047389984, "median": 0.0795329213142395, "p05": 0.050312869250774384, "p95": 0.14976806938648224 }, { "label": "completed_group_1", "mean": 0.05770102143287659, "median": 0.056408509612083435, "p05": 0.04085284471511841, "p95": 0.07934759557247162 }, { "label": "completed_group_2", "mean": 0.05105586349964142, "median": 0.049396127462387085, "p05": 0.034691184759140015, "p95": 0.07344280183315277 }, { "label": "completed_group_3", "mean": 0.039375390857458115, "median": 0.036708764731884, "p05": 0.024915706366300583, "p95": 0.05899723991751671 }, { "label": "completed_group_4", "mean": 0.04410157352685928, "median": 0.04195208102464676, "p05": 0.027429571375250816, "p95": 0.06846393644809723 }, { "label": "completed_group_5", "mean": 0.04500962793827057, "median": 0.043660376220941544, "p05": 0.030392050743103027, "p95": 0.06387433409690857 }, { "label": "completed_group_6", "mean": 0.2357216626405716, "median": 0.2248685657978058, "p05": 0.11950507014989853, "p95": 0.3818710744380951 }, { "label": "current_group_partial", "mean": 0.4404740333557129, "median": 0.4484519362449646, "p05": 0.24098733067512512, "p95": 0.6138508915901184 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8469831943511963, "group": 7, "latest_source_mass": 0.33607518672943115, "layer": 27, "max_source_mass": 0.33607518672943115, "mixer_index": 52, "normalized_entropy": 0.8882111650319092, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10824646055698395, "median": 0.11040513217449188, "p05": 0.05085912346839905, "p95": 0.16382771730422974 }, { "label": "completed_group_1", "mean": 0.09116974472999573, "median": 0.09135521948337555, "p05": 0.056257300078868866, "p95": 0.1266864538192749 }, { "label": "completed_group_2", "mean": 0.0701109915971756, "median": 0.06960364431142807, "p05": 0.05100060999393463, "p95": 0.0908425822854042 }, { "label": "completed_group_3", "mean": 0.0629766583442688, "median": 0.06064727157354355, "p05": 0.04317091405391693, "p95": 0.08672259747982025 }, { "label": "completed_group_4", "mean": 0.07049155235290527, "median": 0.06874367594718933, "p05": 0.050685517489910126, "p95": 0.09697030484676361 }, { "label": "completed_group_5", "mean": 0.07297641038894653, "median": 0.07156282663345337, "p05": 0.051110126078128815, "p95": 0.09974705427885056 }, { "label": "completed_group_6", "mean": 0.18795298039913177, "median": 0.1818247139453888, "p05": 0.1166272684931755, "p95": 0.27661576867103577 }, { "label": "current_group_partial", "mean": 0.33607518672943115, "median": 0.3355332612991333, "p05": 0.20117856562137604, "p95": 0.4717060327529907 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6590619087219238, "group": 7, "latest_source_mass": 0.4179539084434509, "layer": 27, "max_source_mass": 0.4179539084434509, "mixer_index": 53, "normalized_entropy": 0.7978401294136326, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09444728493690491, "median": 0.09060665965080261, "p05": 0.047914937138557434, "p95": 0.15662360191345215 }, { "label": "completed_group_1", "mean": 0.0654030293226242, "median": 0.06396090984344482, "p05": 0.04467916116118431, "p95": 0.09097769111394882 }, { "label": "completed_group_2", "mean": 0.060137998312711716, "median": 0.05839194357395172, "p05": 0.04035535454750061, "p95": 0.08539871126413345 }, { "label": "completed_group_3", "mean": 0.0449567511677742, "median": 0.04218417406082153, "p05": 0.02788582444190979, "p95": 0.06733109056949615 }, { "label": "completed_group_4", "mean": 0.044405773282051086, "median": 0.04241851717233658, "p05": 0.02826966717839241, "p95": 0.06812064349651337 }, { "label": "completed_group_5", "mean": 0.05244143307209015, "median": 0.05142951011657715, "p05": 0.035461653023958206, "p95": 0.07271309196949005 }, { "label": "completed_group_6", "mean": 0.2202538251876831, "median": 0.21207934617996216, "p05": 0.11892802268266678, "p95": 0.34954455494880676 }, { "label": "current_group_partial", "mean": 0.4179539084434509, "median": 0.4249612092971802, "p05": 0.2265779674053192, "p95": 0.586824893951416 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.884438157081604, "group": 7, "latest_source_mass": 0.2829248011112213, "layer": 28, "max_source_mass": 0.2829248011112213, "mixer_index": 54, "normalized_entropy": 0.9062231946945226, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11613816022872925, "median": 0.12147872895002365, "p05": 0.04899268597364426, "p95": 0.1770799458026886 }, { "label": "completed_group_1", "mean": 0.09775452315807343, "median": 0.09778773784637451, "p05": 0.05219751223921776, "p95": 0.14079101383686066 }, { "label": "completed_group_2", "mean": 0.08394932001829147, "median": 0.08419600129127502, "p05": 0.05889316648244858, "p95": 0.10900868475437164 }, { "label": "completed_group_3", "mean": 0.07057419419288635, "median": 0.06822949647903442, "p05": 0.05265102535486221, "p95": 0.0926743820309639 }, { "label": "completed_group_4", "mean": 0.06920696794986725, "median": 0.06799854338169098, "p05": 0.05105053633451462, "p95": 0.09136901795864105 }, { "label": "completed_group_5", "mean": 0.06340071558952332, "median": 0.0628691017627716, "p05": 0.0437660738825798, "p95": 0.0860220268368721 }, { "label": "completed_group_6", "mean": 0.21605131030082703, "median": 0.21297892928123474, "p05": 0.14316481351852417, "p95": 0.2999357581138611 }, { "label": "current_group_partial", "mean": 0.2829248011112213, "median": 0.2790382504463196, "p05": 0.14648588001728058, "p95": 0.4251174330711365 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7424441576004028, "group": 7, "latest_source_mass": 0.278835266828537, "layer": 28, "max_source_mass": 0.3030073344707489, "mixer_index": 55, "normalized_entropy": 0.8379385150653496, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08763164281845093, "median": 0.05824008211493492, "p05": 0.027796253561973572, "p95": 0.2880856990814209 }, { "label": "completed_group_1", "mean": 0.07771545648574829, "median": 0.07654327154159546, "p05": 0.046035416424274445, "p95": 0.11438068747520447 }, { "label": "completed_group_2", "mean": 0.07502467930316925, "median": 0.07301308214664459, "p05": 0.04458038881421089, "p95": 0.11157117038965225 }, { "label": "completed_group_3", "mean": 0.06006643548607826, "median": 0.05717916786670685, "p05": 0.036717794835567474, "p95": 0.08720126748085022 }, { "label": "completed_group_4", "mean": 0.06314835697412491, "median": 0.061049528419971466, "p05": 0.03865773230791092, "p95": 0.09540070593357086 }, { "label": "completed_group_5", "mean": 0.054570816457271576, "median": 0.05328378826379776, "p05": 0.03375566378235817, "p95": 0.07989827543497086 }, { "label": "completed_group_6", "mean": 0.3030073344707489, "median": 0.30074357986450195, "p05": 0.14776623249053955, "p95": 0.469026654958725 }, { "label": "current_group_partial", "mean": 0.278835266828537, "median": 0.26309943199157715, "p05": 0.10582277178764343, "p95": 0.5049530863761902 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8953055143356323, "group": 8, "latest_source_mass": 0.18233531713485718, "layer": 29, "max_source_mass": 0.2779633104801178, "mixer_index": 56, "normalized_entropy": 0.911449288833841, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1135263666510582, "median": 0.0953834131360054, "p05": 0.02703639306128025, "p95": 0.31321749091148376 }, { "label": "completed_group_1", "mean": 0.10274903476238251, "median": 0.10363410413265228, "p05": 0.06506763398647308, "p95": 0.13818706572055817 }, { "label": "completed_group_2", "mean": 0.0858360230922699, "median": 0.08501487225294113, "p05": 0.059830889105796814, "p95": 0.11487221717834473 }, { "label": "completed_group_3", "mean": 0.08475606143474579, "median": 0.08265408128499985, "p05": 0.05679357051849365, "p95": 0.11555421352386475 }, { "label": "completed_group_4", "mean": 0.08078759908676147, "median": 0.07859720289707184, "p05": 0.055059194564819336, "p95": 0.11340422928333282 }, { "label": "completed_group_5", "mean": 0.07204629480838776, "median": 0.07071015238761902, "p05": 0.0498509518802166, "p95": 0.09922727942466736 }, { "label": "completed_group_6", "mean": 0.2779633104801178, "median": 0.2749147415161133, "p05": 0.15705733001232147, "p95": 0.39374127984046936 }, { "label": "completed_group_7", "mean": 0.18233531713485718, "median": 0.16221198439598083, "p05": 0.07201667875051498, "p95": 0.35018864274024963 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.801485300064087, "group": 8, "latest_source_mass": 0.3215197026729584, "layer": 29, "max_source_mass": 0.3215197026729584, "mixer_index": 57, "normalized_entropy": 0.8198912931549752, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07824120670557022, "median": 0.06902759522199631, "p05": 0.017845120280981064, "p95": 0.17217300832271576 }, { "label": "completed_group_1", "mean": 0.060231711715459824, "median": 0.05764210224151611, "p05": 0.03765189275145531, "p95": 0.0922410786151886 }, { "label": "completed_group_2", "mean": 0.054100219160318375, "median": 0.0524706169962883, "p05": 0.034336626529693604, "p95": 0.07949143648147583 }, { "label": "completed_group_3", "mean": 0.039871927350759506, "median": 0.037151023745536804, "p05": 0.0235070139169693, "p95": 0.060298819094896317 }, { "label": "completed_group_4", "mean": 0.04207881912589073, "median": 0.040273066610097885, "p05": 0.021681729704141617, "p95": 0.06873302906751633 }, { "label": "completed_group_5", "mean": 0.045095667243003845, "median": 0.044285234063863754, "p05": 0.025416042655706406, "p95": 0.06805071234703064 }, { "label": "completed_group_6", "mean": 0.2047339677810669, "median": 0.1937887817621231, "p05": 0.08607597649097443, "p95": 0.3561563789844513 }, { "label": "completed_group_7", "mean": 0.15412676334381104, "median": 0.13271009922027588, "p05": 0.05149175971746445, "p95": 0.32324373722076416 }, { "label": "current_group_partial", "mean": 0.3215197026729584, "median": 0.31218427419662476, "p05": 0.08214674144983292, "p95": 0.5716250538825989 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.008213520050049, "group": 8, "latest_source_mass": 0.1996053159236908, "layer": 30, "max_source_mass": 0.21496367454528809, "mixer_index": 58, "normalized_entropy": 0.9139773606959576, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09438058733940125, "median": 0.09028880298137665, "p05": 0.02690170146524906, "p95": 0.18525993824005127 }, { "label": "completed_group_1", "mean": 0.07153058052062988, "median": 0.070882648229599, "p05": 0.041449472308158875, "p95": 0.10379751026630402 }, { "label": "completed_group_2", "mean": 0.07423336803913116, "median": 0.0730072408914566, "p05": 0.048166655004024506, "p95": 0.10590565949678421 }, { "label": "completed_group_3", "mean": 0.06403130292892456, "median": 0.06157837435603142, "p05": 0.04371092468500137, "p95": 0.08951176702976227 }, { "label": "completed_group_4", "mean": 0.0645662397146225, "median": 0.0623253732919693, "p05": 0.043447867035865784, "p95": 0.09214113652706146 }, { "label": "completed_group_5", "mean": 0.05577655881643295, "median": 0.055099695920944214, "p05": 0.03416105732321739, "p95": 0.079859159886837 }, { "label": "completed_group_6", "mean": 0.21496367454528809, "median": 0.2118406593799591, "p05": 0.13403329253196716, "p95": 0.31010040640830994 }, { "label": "completed_group_7", "mean": 0.1609123945236206, "median": 0.15419161319732666, "p05": 0.07973575592041016, "p95": 0.27241089940071106 }, { "label": "current_group_partial", "mean": 0.1996053159236908, "median": 0.19533705711364746, "p05": 0.0843900740146637, "p95": 0.33647382259368896 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.8966764211654663, "group": 8, "latest_source_mass": 0.21615305542945862, "layer": 30, "max_source_mass": 0.21615305542945862, "mixer_index": 59, "normalized_entropy": 0.8632146393815058, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09610478579998016, "median": 0.0941460132598877, "p05": 0.030834276229143143, "p95": 0.17860566079616547 }, { "label": "completed_group_1", "mean": 0.07267358899116516, "median": 0.06876465678215027, "p05": 0.03731950744986534, "p95": 0.11969097703695297 }, { "label": "completed_group_2", "mean": 0.060228411108255386, "median": 0.05793646723031998, "p05": 0.0349978432059288, "p95": 0.09366682916879654 }, { "label": "completed_group_3", "mean": 0.046177469193935394, "median": 0.04286886006593704, "p05": 0.02664034441113472, "p95": 0.07394801080226898 }, { "label": "completed_group_4", "mean": 0.04206448048353195, "median": 0.03912021219730377, "p05": 0.022682487964630127, "p95": 0.07153408974409103 }, { "label": "completed_group_5", "mean": 0.04096413776278496, "median": 0.039336949586868286, "p05": 0.023794004693627357, "p95": 0.06405855715274811 }, { "label": "completed_group_6", "mean": 0.21562576293945312, "median": 0.20660808682441711, "p05": 0.10648336261510849, "p95": 0.3508927822113037 }, { "label": "completed_group_7", "mean": 0.21000832319259644, "median": 0.204543337225914, "p05": 0.08861437439918518, "p95": 0.3495482802391052 }, { "label": "current_group_partial", "mean": 0.21615305542945862, "median": 0.19288693368434906, "p05": 0.07421350479125977, "p95": 0.43364205956459045 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.981720209121704, "group": 8, "latest_source_mass": 0.17098912596702576, "layer": 31, "max_source_mass": 0.23084521293640137, "mixer_index": 60, "normalized_entropy": 0.9019197352708571, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10501759499311447, "median": 0.10529834032058716, "p05": 0.012508269399404526, "p95": 0.22564347088336945 }, { "label": "completed_group_1", "mean": 0.06946458667516708, "median": 0.06936883926391602, "p05": 0.025395508855581284, "p95": 0.11417273432016373 }, { "label": "completed_group_2", "mean": 0.07816962152719498, "median": 0.07627303898334503, "p05": 0.042986176908016205, "p95": 0.11851722747087479 }, { "label": "completed_group_3", "mean": 0.07385894656181335, "median": 0.07061562687158585, "p05": 0.04739787429571152, "p95": 0.1065390408039093 }, { "label": "completed_group_4", "mean": 0.06364607810974121, "median": 0.06080539897084236, "p05": 0.0418400801718235, "p95": 0.09455618262290955 }, { "label": "completed_group_5", "mean": 0.05328303202986717, "median": 0.05173730105161667, "p05": 0.03466875106096268, "p95": 0.07786844670772552 }, { "label": "completed_group_6", "mean": 0.23084521293640137, "median": 0.2265024036169052, "p05": 0.1294252872467041, "p95": 0.3483029305934906 }, { "label": "completed_group_7", "mean": 0.1547257900238037, "median": 0.15095551311969757, "p05": 0.057022616267204285, "p95": 0.2705056667327881 }, { "label": "current_group_partial", "mean": 0.17098912596702576, "median": 0.14655888080596924, "p05": 0.04728491231799126, "p95": 0.3703039288520813 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0028607845306396, "group": 8, "latest_source_mass": 0.12469469010829926, "layer": 31, "max_source_mass": 0.2620389461517334, "mixer_index": 61, "normalized_entropy": 0.911541225776195, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.0842779129743576, "median": 0.06333573907613754, "p05": 0.03144961595535278, "p95": 0.24860140681266785 }, { "label": "completed_group_1", "mean": 0.10298711061477661, "median": 0.10017631947994232, "p05": 0.06670551002025604, "p95": 0.14914412796497345 }, { "label": "completed_group_2", "mean": 0.09207960218191147, "median": 0.09137493371963501, "p05": 0.056820835918188095, "p95": 0.13043460249900818 }, { "label": "completed_group_3", "mean": 0.07771934568881989, "median": 0.0746275931596756, "p05": 0.04656440392136574, "p95": 0.11502407491207123 }, { "label": "completed_group_4", "mean": 0.06265503168106079, "median": 0.0610337033867836, "p05": 0.038839153945446014, "p95": 0.09315080940723419 }, { "label": "completed_group_5", "mean": 0.055270493030548096, "median": 0.0542943961918354, "p05": 0.035113874822854996, "p95": 0.07895103842020035 }, { "label": "completed_group_6", "mean": 0.2620389461517334, "median": 0.25993645191192627, "p05": 0.13434287905693054, "p95": 0.3996090292930603 }, { "label": "completed_group_7", "mean": 0.1382768601179123, "median": 0.12482025474309921, "p05": 0.058253247290849686, "p95": 0.2677440643310547 }, { "label": "current_group_partial", "mean": 0.12469469010829926, "median": 0.11286871135234833, "p05": 0.04615473002195358, "p95": 0.24487172067165375 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0655057430267334, "group": 8, "latest_source_mass": 0.09924688190221786, "layer": 32, "max_source_mass": 0.21871674060821533, "mixer_index": 62, "normalized_entropy": 0.9400521750629723, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10472239553928375, "median": 0.08473113924264908, "p05": 0.030345402657985687, "p95": 0.2588862180709839 }, { "label": "completed_group_1", "mean": 0.11792458593845367, "median": 0.11655837297439575, "p05": 0.07099610567092896, "p95": 0.17131033539772034 }, { "label": "completed_group_2", "mean": 0.0928182601928711, "median": 0.09008213877677917, "p05": 0.06447342038154602, "p95": 0.12974750995635986 }, { "label": "completed_group_3", "mean": 0.10183011740446091, "median": 0.10004840791225433, "p05": 0.0707390084862709, "p95": 0.1374078094959259 }, { "label": "completed_group_4", "mean": 0.08287280797958374, "median": 0.08090724050998688, "p05": 0.05679214745759964, "p95": 0.11534547060728073 }, { "label": "completed_group_5", "mean": 0.06570011377334595, "median": 0.06424097716808319, "p05": 0.04414675012230873, "p95": 0.09294299781322479 }, { "label": "completed_group_6", "mean": 0.21871674060821533, "median": 0.21457019448280334, "p05": 0.13963721692562103, "p95": 0.3112483620643616 }, { "label": "completed_group_7", "mean": 0.11616809666156769, "median": 0.10701116174459457, "p05": 0.05174187570810318, "p95": 0.21264401078224182 }, { "label": "current_group_partial", "mean": 0.09924688190221786, "median": 0.08725139498710632, "p05": 0.032173603773117065, "p95": 0.20341788232326508 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0258567333221436, "group": 8, "latest_source_mass": 0.12343063950538635, "layer": 32, "max_source_mass": 0.22868779301643372, "mixer_index": 63, "normalized_entropy": 0.9220071330979595, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09674200415611267, "median": 0.07686861604452133, "p05": 0.04638649895787239, "p95": 0.2122267782688141 }, { "label": "completed_group_1", "mean": 0.09482249617576599, "median": 0.09244489669799805, "p05": 0.06637479364871979, "p95": 0.13019198179244995 }, { "label": "completed_group_2", "mean": 0.08996410667896271, "median": 0.08937586098909378, "p05": 0.06193297728896141, "p95": 0.12055040895938873 }, { "label": "completed_group_3", "mean": 0.06836152076721191, "median": 0.06606560945510864, "p05": 0.04674140736460686, "p95": 0.09358662366867065 }, { "label": "completed_group_4", "mean": 0.06293784081935883, "median": 0.061281464993953705, "p05": 0.04231695458292961, "p95": 0.08961650729179382 }, { "label": "completed_group_5", "mean": 0.05016915872693062, "median": 0.049648597836494446, "p05": 0.03570414334535599, "p95": 0.06624776124954224 }, { "label": "completed_group_6", "mean": 0.22868779301643372, "median": 0.2242993861436844, "p05": 0.1197199895977974, "p95": 0.3530009686946869 }, { "label": "completed_group_7", "mean": 0.1848844289779663, "median": 0.17580115795135498, "p05": 0.10029035806655884, "p95": 0.3003215491771698 }, { "label": "current_group_partial", "mean": 0.12343063950538635, "median": 0.10480014234781265, "p05": 0.05131315439939499, "p95": 0.2450288087129593 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.7394249439239502, "group": null, "latest_source_mass": 0.2556654214859009, "layer": null, "max_source_mass": 0.3314756751060486, "mixer_index": null, "normalized_entropy": 0.7916464078663832, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.10602210462093353, "median": 0.10817556083202362, "p05": 0.03750653192400932, "p95": 0.16905617713928223 }, { "label": "completed_group_1", "mean": 0.05902107432484627, "median": 0.05735805630683899, "p05": 0.039041999727487564, "p95": 0.08384283632040024 }, { "label": "completed_group_2", "mean": 0.049884356558322906, "median": 0.04808305203914642, "p05": 0.03180687129497528, "p95": 0.07381068915128708 }, { "label": "completed_group_3", "mean": 0.034865088760852814, "median": 0.03287554159760475, "p05": 0.02153801918029785, "p95": 0.05384448915719986 }, { "label": "completed_group_4", "mean": 0.03282830864191055, "median": 0.0315137580037117, "p05": 0.020427687093615532, "p95": 0.050021201372146606 }, { "label": "completed_group_5", "mean": 0.043302543461322784, "median": 0.041625380516052246, "p05": 0.027595922350883484, "p95": 0.06415434926748276 }, { "label": "completed_group_6", "mean": 0.08693542331457138, "median": 0.07748770713806152, "p05": 0.03693632781505585, "p95": 0.17101936042308807 }, { "label": "completed_group_7", "mean": 0.3314756751060486, "median": 0.32931262254714966, "p05": 0.0907490998506546, "p95": 0.5930418968200684 }, { "label": "completed_group_8", "mean": 0.2556654214859009, "median": 0.23854607343673706, "p05": 0.04562881588935852, "p95": 0.5189820528030396 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", "optimizer_state_before": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08528144657611847, 0.2063225358724594, 0.07479097694158554, 0.053203217685222626, 0.1070452407002449, 0.03943018242716789, 0.041509829461574554, 0.06257687509059906, 0.061002109199762344, 0.04113767296075821, 0.040399398654699326, 0.03907898813486099, 0.06714949756860733, 0.04437125846743584, 0.0412614606320858, 0.046277787536382675, 0.10073095560073853, 0.05776162073016167, 0.05185067281126976, 0.07827401161193848, 0.04170985147356987, 0.041845910251140594, 0.038240060210227966, 0.046402644366025925, 0.030182205140590668, 0.025695007294416428, 0.024306239560246468, 0.017157377675175667, 0.030125634744763374, 0.01829664409160614, 0.024464696645736694, 0.032671861350536346 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1464898420854297e-05, "first_to_last_ratio": 0.8455323660117368, "imbalance_abs_log_ratio": 0.16778883105478307, "last_quartile_mean": 1.355938445612992e-05, "mean": 1.3058880313110421e-05, "normalized": [ 0.8914129879074366, 1.077571175585955, 0.9537440677326601, 0.8574453930361242, 0.9048950001587052, 0.8425968627438281, 0.7839582579962869, 0.7118870894358409, 0.5788084225246481, 0.530693141917732, 0.5023627734883869, 0.4844869540829288, 0.6659476097626051, 0.6212422516778212, 0.5836177358524666, 0.5530522631862488, 0.6705030645582619, 0.6390296210597277, 0.6539820627182751, 0.682372291616315, 3.0705747439001825, 2.5957154007666214, 2.1136981524200404, 1.7237888584415353, 1.9773452676094638, 1.7509357725203292, 1.3561771885891878, 0.9354730786944212, 0.8232148160475435, 0.5856284416831322, 0.45203530207589593, 0.4258039502093924 ], "peak_layer": 21, "peak_normalized": 3.0705747439001825, "population_cv": 0.6442856671110848, "rest_mean": 9.924206164281781e-06, "spike_contrast": 3.0215132797934157, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.998612071678508e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.1640855518635362e-05, 1.4071873010834679e-05, 1.2454829629859887e-05, 1.119727676268667e-05, 1.1816915503004566e-05, 1.100337158277398e-05, 1.0237617061648052e-05, 9.29644829739118e-06, 7.558589913969627e-06, 6.930258223292185e-06, 6.560295332747046e-06, 6.326857146632392e-06, 8.696530130691826e-06, 8.112728210107889e-06, 7.6213941611058544e-06, 7.222243311844068e-06, 8.756019269640092e-06, 8.345011337951291e-06, 8.540273483959027e-06, 8.91101808520034e-06, 4.009826807305217e-05, 3.389713674550876e-05, 2.7602531190495938e-05, 2.2510752387461253e-05, 2.5821915187407285e-05, 2.2865260689286515e-05, 1.7710155589156784e-05, 1.2216230970807374e-05, 1.0750263754744083e-05, 7.64765172789339e-06, 5.90307490710984e-06, 5.560522822634084e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.568297117657494e-05, "first_to_last_ratio": 0.8467277881503474, "imbalance_abs_log_ratio": 0.16637601951453293, "last_quartile_mean": 5.3952370308252284e-05, "mean": 5.2044935273443116e-05, "normalized": [ 0.8954256553649687, 1.0662758237422765, 0.9521993752570108, 0.8636540973713456, 0.9057610370429903, 0.8375327549345261, 0.7860647798171227, 0.7151678277570597, 0.5850341337918961, 0.5346127123576437, 0.503507273341017, 0.4847402912324663, 0.6665718638058081, 0.6225508325294531, 0.5847778078263528, 0.5518762696320233, 0.6703631394314256, 0.6385628522390465, 0.6481962908175356, 0.6867108224315941, 3.0618207183884687, 2.6065455407648237, 2.120445260989786, 1.7184046595544948, 1.970749079693583, 1.7371379145623163, 1.3426067482179225, 0.9360274925849964, 0.8279776750708923, 0.5898304420688857, 0.45715187233946364, 0.4317169550408049 ], "peak_layer": 21, "peak_normalized": 3.0618207183884687, "population_cv": 0.6425897668413451, "rest_mean": 3.955807295386438e-05, "spike_contrast": 3.0202176920627024, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011947399179916829, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.6602370275650173e-05, 5.549425623030402e-05, 4.95571548526641e-05, 4.494882159633562e-05, 4.714027454610914e-05, 4.35893380199559e-05, 4.091069058631547e-05, 3.7220863305265084e-05, 3.044806362595409e-05, 2.7823884011013433e-05, 2.620500345074106e-05, 2.5228277081623673e-05, 3.4691689506871626e-05, 3.240061778342351e-05, 3.0434723157668486e-05, 2.872236473194789e-05, 3.4889006201410666e-05, 3.32339623128064e-05, 3.373533400008455e-05, 3.5739820305025205e-05, 0.00015935226110741496, 0.00013565749395638704, 0.00011035843635909259, 8.943425928009674e-05, 0.00010256750829285011, 9.04092303244397e-05, 6.987588130868971e-05, 4.8715490265749395e-05, 4.30920445069205e-05, 3.0697687179781497e-05, 2.3792439606040716e-05, 2.246868098154664e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.040512115228921175, "first_to_last_ratio": 0.8467277882597376, "imbalance_abs_log_ratio": 0.16637601938534127, "last_quartile_mean": 0.047845500987023115, "mean": 0.046153969364240766, "normalized": [ 0.8954256214236297, 1.0662758259809275, 0.9521993286286553, 0.8636540898502394, 0.9057610206102777, 0.8375327282419698, 0.7860647826245731, 0.7151678297957529, 0.5850341696121162, 0.5346127266472548, 0.5035072907230911, 0.484740303083307, 0.666571864231768, 0.6225508316512409, 0.5847778206533832, 0.5518762470155281, 0.6703631006740365, 0.6385628347715187, 0.6481963011119384, 0.6867108353729822, 3.0618206630891383, 2.6065457195235173, 2.120445297956706, 1.7184047348201537, 1.9707489675025585, 1.7371379031231275, 1.3426067663321783, 0.9360274706240039, 0.8279776759485932, 0.5898304215013819, 0.45715187640138183, 0.4317169504730694 ], "peak_layer": 21, "peak_normalized": 3.0618206630891383, "population_cv": 0.6425897745941355, "rest_mean": 0.03508049470406992, "spike_contrast": 3.0202177427352903, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10595073252916336, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.041327446699142456, 0.04921286180615425, 0.04394777864217758, 0.03986106440424919, 0.041804466396570206, 0.03865545988082886, 0.036280009895563126, 0.03300783410668373, 0.027001649141311646, 0.02467449940741062, 0.023238860070705414, 0.022372689098119736, 0.03076493740081787, 0.02873319201171398, 0.02698981761932373, 0.02547127939760685, 0.030939918011426926, 0.02947220951318741, 0.029916832223534584, 0.031694430857896805, 0.14131517708301544, 0.12030243128538132, 0.0978669673204422, 0.07931119948625565, 0.0909578874707222, 0.08017580956220627, 0.061966631561517715, 0.04320138320326805, 0.03821445629000664, 0.027223015204072, 0.021099373698234558, 0.019925450906157494 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8178302667593016e-06, "first_to_last_ratio": 0.5015852279301855, "imbalance_abs_log_ratio": 0.6899817399970322, "last_quartile_mean": 3.6241702616734983e-06, "mean": 2.789410469716813e-06, "normalized": [ 0.8037687132877044, 0.7183737285046393, 0.691319672118338, 0.6807254875963729, 0.6052756387601809, 0.5818941580160492, 0.565889251881305, 0.5662715078678539, 0.4369805018739323, 0.41687613565271625, 0.40783237228271835, 0.41545507589336067, 0.5056083509177715, 0.5034210266522742, 0.5059866535158252, 0.503281516870639, 0.5409761795262856, 0.5584199152903219, 0.570327765976957, 0.6037643031707576, 2.871505340690978, 2.8594035701851728, 2.496246607238993, 2.1963141894218743, 2.393616152828075, 2.14842944375899, 1.7201180299928378, 1.2992827591669334, 1.010592310992817, 0.7253855724579202, 0.5639386418465228, 0.532719425762883 ], "peak_layer": 21, "peak_normalized": 2.871505340690978, "population_cv": 0.7722487160005334, "rest_mean": 1.9818156126213445e-06, "spike_contrast": 3.6080161305089784, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.150422698032344e-06, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.242040864075534e-06, 2.0038391994603444e-06, 1.9283743313280866e-06, 1.8988228021044051e-06, 1.6883622038221802e-06, 1.6231416566370171e-06, 1.5784974038979271e-06, 1.5795636727489182e-06, 1.2189179869892541e-06, 1.162838657364773e-06, 1.1376118891348597e-06, 1.1588747383939335e-06, 1.4103492276262841e-06, 1.4042478824194404e-06, 1.4114044688540162e-06, 1.4038587323739193e-06, 1.5090046190380235e-06, 1.5576623582091997e-06, 1.5908782415863243e-06, 1.6841464685057872e-06, 8.009807061171159e-06, 7.976050255820155e-06, 6.96305642122752e-06, 6.126421794760972e-06, 6.676777957181912e-06, 5.9928515838691965e-06, 4.798115242010681e-06, 3.6242329315427924e-06, 2.818956772898673e-06, 2.0233981103956467e-06, 1.5730563518445706e-06, 1.4859731436445145e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.489204118523048e-05, "first_to_last_ratio": 1.0102007473277157, "imbalance_abs_log_ratio": 0.01014907083348142, "last_quartile_mean": 3.45397103274081e-05, "mean": 3.719139070312849e-05, "normalized": [ 0.978746778311648, 1.126838378619792, 1.0180419362177582, 0.9373857091569082, 0.9573174487244173, 0.8927610480088312, 0.8376961987226147, 0.7566142687378048, 0.6421710022561828, 0.5898929811647327, 0.5545314669590201, 0.532480114188549, 0.7191190555706168, 0.6685869838104541, 0.6276517153974418, 0.5907065315366481, 0.7404063048506205, 0.7009976783342637, 0.7076013069062036, 0.7352296418703005, 3.079471174742761, 2.569668642671924, 2.031249327096867, 1.5752201563285502, 1.8426406293315767, 1.5558607485184148, 1.2001391008391664, 0.8699295985429139, 0.6815919538112388, 0.4881760018644189, 0.4044368033178356, 0.3868393135895246 ], "peak_layer": 21, "peak_normalized": 3.079471174742761, "population_cv": 0.6121845820849332, "rest_mean": 2.8791301971338084e-05, "spike_contrast": 2.867250322231956, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.255186985479668e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.640095383161679e-05, 4.1908686398528516e-05, 3.786239540204406e-05, 3.486267814878374e-05, 3.560396726243198e-05, 3.320302494103089e-05, 3.115508661721833e-05, 2.8139536880189553e-05, 2.3883232643129304e-05, 2.1938940335530788e-05, 2.0623796444851905e-05, 1.98036759684328e-05, 2.674503775779158e-05, 2.4865679733920842e-05, 2.3343240172835067e-05, 2.196919740526937e-05, 2.7536740162759088e-05, 2.6071078536915593e-05, 2.631667666719295e-05, 2.7344212867319584e-05, 0.00011452981561888009, 9.55695504671894e-05, 7.554498733952641e-05, 5.858462827745825e-05, 6.853036757092923e-05, 5.786462497781031e-05, 4.463484219741076e-05, 3.235389158362523e-05, 2.5349352654302493e-05, 1.8155944417230785e-05, 1.5041567166917957e-05, 1.4387092051038053e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6185193746641744e-05, "first_to_last_ratio": 1.1813045118653667, "imbalance_abs_log_ratio": 0.16661934637193965, "last_quartile_mean": 2.2166336862028402e-05, "mean": 2.6416844121968097e-05, "normalized": [ 1.0582483794330777, 1.1839113566742743, 1.0890809034246702, 1.013584100454415, 1.0023428903115454, 0.935888205711016, 0.8664359614724408, 0.7803558787188484, 0.6948576391403528, 0.6454578452882075, 0.6088646607200737, 0.5814486923097771, 0.764934624577467, 0.7118430529936276, 0.6645388585723574, 0.629085742341511, 0.8117172258565646, 0.7682825984070001, 0.7655811246450389, 0.7857710139637165, 3.1099202343035985, 2.532060860240954, 1.9118190653037337, 1.3711803024378004, 1.6503311681253898, 1.3613514907810353, 1.0865408308937656, 0.8606461743085547, 0.5744935662233659, 0.4301536296331042, 0.37929000098103194, 0.3699819217516835 ], "peak_layer": 21, "peak_normalized": 3.1099202343035985, "population_cv": 0.583027433434022, "rest_mean": 2.096195011189492e-05, "spike_contrast": 2.6654615376008266, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5873271776363254e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.795558248180896e-05, 3.127520176349208e-05, 2.8770080461981706e-05, 2.6775693186209537e-05, 2.6478735890123062e-05, 2.4723212845856324e-05, 2.2888503735885024e-05, 2.061453960777726e-05, 1.835594594012946e-05, 1.7050959286279976e-05, 1.608428283361718e-05, 1.5360039469669573e-05, 2.020715874095913e-05, 1.8804646970238537e-05, 1.755501943989657e-05, 1.661845999478828e-05, 2.1443007426569238e-05, 2.0295601643738337e-05, 2.0224237232469022e-05, 2.0757590391440317e-05, 8.215427806135267e-05, 6.688905705232173e-05, 5.050422623753548e-05, 3.6222256312612444e-05, 4.3596541217993945e-05, 3.59626101271715e-05, 2.8702979761874303e-05, 2.2735555830877274e-05, 1.5176306987996213e-05, 1.136330138251651e-05, 1.0019644832937047e-05, 9.77375475486042e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.068558847502572e-05, "first_to_last_ratio": 0.7914793598068353, "imbalance_abs_log_ratio": 0.2338514773139948, "last_quartile_mean": 0.00011457732580311131, "mean": 0.00010754580421235005, "normalized": [ 0.8538738720411836, 1.0095472825344882, 0.8946236658075398, 0.8160443171140711, 0.8628188117624934, 0.8243645351821149, 0.7801447421092808, 0.7044034320375209, 0.5665887799168225, 0.525608047514618, 0.4958157795364288, 0.4792984745903937, 0.6722545548598279, 0.6294814179611099, 0.5936561952518512, 0.5594680423070397, 0.6646720414600984, 0.6282112723775689, 0.6412680496471301, 0.6738175767739937, 3.007412745002923, 2.616093975094252, 2.1689004360837822, 1.8085787931443134, 2.061415837562002, 1.8302883103495597, 1.4233838379602504, 0.9902275928882596, 0.8088632655175125, 0.5559016015107706, 0.43871981288186035, 0.4142529012189394 ], "peak_layer": 21, "peak_normalized": 3.007412745002923, "population_cv": 0.6573728454605334, "rest_mean": 8.100827242877265e-05, "spike_contrast": 3.096578513808316, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000250848475843668, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 9.183055226458237e-05, 0.00010857257439056411, 9.621302160667256e-05, 8.776214235695079e-05, 9.279254300054163e-05, 8.865694690030068e-05, 8.390129369217902e-05, 7.575563358841464e-05, 6.093424599384889e-05, 5.6526940170442685e-05, 5.3322906751418486e-05, 5.1546539907576516e-05, 7.229815673781559e-05, 6.769808533135802e-05, 6.384523294400424e-05, 6.016844054101966e-05, 7.148268923629075e-05, 6.756148650310934e-05, 6.896568811498582e-05, 7.246625318657607e-05, 0.00032343462225981057, 0.00028134993044659495, 0.00023325614165514708, 0.00019450506079010665, 0.00022169662406668067, 0.00019683982827700675, 0.00015307895955629647, 0.00010649482283042744, 8.698985038790852e-05, 5.978488479740918e-05, 4.7182475100271404e-05, 4.455116140889004e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13266965746879578, 0.045500293374061584, 0.018729008734226227, 0.03815881907939911, 0.02209169790148735, 0.01640710048377514, 0.02340921014547348, 0.05481670796871185, 0.037687551230192184, 0.04149388521909714, 0.048693906515836716, 0.06190986558794975, 0.04065176844596863, 0.049332331866025925, 0.0589165985584259, 0.08173932880163193, 0.043543290346860886, 0.05228494107723236, 0.04403071105480194, 0.0687810555100441, 0.030229199677705765, 0.04699210822582245, 0.06082639843225479, 0.07667754590511322, 0.050203632563352585, 0.05539015680551529, 0.06946423649787903, 0.07630576938390732, 0.07239200174808502, 0.09628339111804962, 0.12468792498111725, 0.15056027472019196 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1373777851986233e-05, "first_to_last_ratio": 1.0318498984683846, "imbalance_abs_log_ratio": 0.03135320926034069, "last_quartile_mean": 1.1022705791674525e-05, "mean": 1.1832125267119409e-05, "normalized": [ 0.9800012201604696, 1.1859002007221142, 1.0527348906308314, 0.9435939146843411, 0.9942658078795703, 0.9213667974504651, 0.8458121244297144, 0.7664247312681599, 0.6009152433386855, 0.5652338271700392, 0.5379286363125223, 0.5220099956402732, 0.7166619388130793, 0.6623381156132455, 0.6162294438595727, 0.5904010132472348, 0.7324356475968063, 0.6882143187892091, 0.7092129547489621, 0.8089361635707597, 3.115397457088153, 2.4321274256844663, 1.9642577727784198, 1.5948693957730091, 1.9522870367789025, 1.5037975945274742, 1.0368577963312928, 0.8807302172788337, 0.6474760441659657, 0.5016437023911682, 0.4711656138967313, 0.45877295737952767 ], "peak_layer": 21, "peak_normalized": 3.115397457088153, "population_cv": 0.603271891680543, "rest_mean": 9.176935405034951e-06, "spike_contrast": 2.851730928390011, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.617015052237548e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1595497198868543e-05, 1.4031719729246106e-05, 1.2456091099011246e-05, 1.1164721399836708e-05, 1.1764277587644756e-05, 1.0901727364398539e-05, 1.000775500870077e-05, 9.068433428183198e-06, 7.110104434104869e-06, 6.687917448289227e-06, 6.364839009620482e-06, 6.176487659104168e-06, 8.479633834213018e-06, 7.836867553123739e-06, 7.291303973033791e-06, 6.985698746575508e-06, 8.66627033246914e-06, 8.143038030539174e-06, 8.391496521653607e-06, 9.571434020472225e-06, 3.686177296913229e-05, 2.8777236366295256e-05, 2.3241344024427235e-05, 1.8870694475481287e-05, 2.309970477654133e-05, 1.7793121514841914e-05, 1.226823133038124e-05, 1.0420910257380456e-05, 7.661017662030645e-06, 5.9355111261538696e-06, 5.574890565185342e-06, 5.428259100881405e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.52927956757776e-05, "first_to_last_ratio": 1.0314200550157464, "imbalance_abs_log_ratio": 0.03093654690539193, "last_quartile_mean": 4.3913045374210924e-05, "mean": 4.717359485084671e-05, "normalized": [ 0.9832135136979314, 1.173164387917132, 1.0505394497504255, 0.9501840008496486, 0.9944561477628384, 0.9147691984159712, 0.8464742625690872, 0.7682410828301822, 0.6067449473404415, 0.5681699639617173, 0.5386389915171271, 0.5211401199814687, 0.7167243803511351, 0.6632392162921593, 0.6172595012574109, 0.588538933455825, 0.7318717018485573, 0.6893632656633977, 0.7026754642712614, 0.8079854981931587, 3.1150142181403937, 2.4538347722826446, 1.966137677039432, 1.5845641436792013, 1.9375330190421414, 1.4892879811658426, 1.035008152926599, 0.8840423036226741, 0.6517018335432763, 0.5065002027138268, 0.477654085237329, 0.4653275826797635 ], "peak_layer": 21, "peak_normalized": 3.1150142181403937, "population_cv": 0.6027267551758898, "rest_mean": 3.659083860704278e-05, "spike_contrast": 2.8509999371074524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010432047856738791, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.638171594706364e-05, 5.534238152904436e-05, 4.9557722377358004e-05, 4.482359508983791e-05, 4.691207141149789e-05, 4.315295154810883e-05, 3.993123391410336e-05, 3.6240693589206785e-05, 2.862234032363631e-05, 2.6802619686350226e-05, 2.5409537556697614e-05, 2.458405288052745e-05, 3.381046553840861e-05, 3.1287378078559414e-05, 2.9118349630152807e-05, 2.7763497200794518e-05, 3.4525019145803526e-05, 3.251974339946173e-05, 3.3147727663163096e-05, 3.8115580537123606e-05, 0.00014694641868118197, 0.00011575620737858117, 9.274978219764307e-05, 7.47495869291015e-05, 9.140039765043184e-05, 7.025506783975288e-05, 4.8825055273482576e-05, 4.1703453462105244e-05, 3.074311825912446e-05, 2.3893435354693793e-05, 2.253266029583756e-05, 2.1951174858259037e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0401661042124033, "first_to_last_ratio": 1.0314200458773342, "imbalance_abs_log_ratio": 0.030936538045362262, "last_quartile_mean": 0.03894252819009125, "mean": 0.041834015981294215, "normalized": [ 0.9832134785727225, 1.1731643857154097, 1.0505394772834944, 0.9501839314909589, 0.9944561321819699, 0.9147691986147233, 0.8464742543512297, 0.7682411398349651, 0.6067449159473433, 0.5681700011232179, 0.5386389788463947, 0.5211401213699254, 0.7167244214988735, 0.663239179094363, 0.6172595023900571, 0.5885389178924195, 0.7318717092957638, 0.6893632192579286, 0.7026754233788964, 0.8079854502108175, 3.1150143563768617, 2.4538347987960454, 1.9661377249833973, 1.5845640989337733, 1.937532949114683, 1.4892880148885772, 1.0350081198083678, 0.8840423776654689, 0.6517018613150984, 0.5065001920223166, 0.4776540925208594, 0.46532757522307666 ], "peak_layer": 21, "peak_normalized": 3.1150143563768617, "population_cv": 0.6027267719407451, "rest_mean": 0.03244912169045872, "spike_contrast": 2.850999975725323, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09251244515180587, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.04113176837563515, 0.04907817766070366, 0.04394828528165817, 0.03975000977516174, 0.04160209372639656, 0.03826846927404404, 0.03541141748428345, 0.03213861212134361, 0.025382576510310173, 0.02376883290708065, 0.02253343164920807, 0.021801384165883064, 0.029983460903167725, 0.027745958417654037, 0.025822443887591362, 0.024620946496725082, 0.030617132782936096, 0.028838831931352615, 0.029395734891295433, 0.03380127623677254, 0.13031356036663055, 0.10265376418828964, 0.08225143700838089, 0.06628867983818054, 0.08105478435754776, 0.06230289861559868, 0.043298546224832535, 0.03698304295539856, 0.027263306081295013, 0.02118893712759018, 0.019982188940048218, 0.019466521218419075 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.809132328389751e-06, "first_to_last_ratio": 0.5997744339081645, "imbalance_abs_log_ratio": 0.5112016379368517, "last_quartile_mean": 3.016354526153009e-06, "mean": 2.487152045205221e-06, "normalized": [ 0.890408531440174, 0.809673961899388, 0.7756270411999202, 0.7664197016433177, 0.6818557608603003, 0.6497146124011205, 0.6267527377763467, 0.6186766662305586, 0.4670749596929491, 0.4522141597899267, 0.449235216497964, 0.45731293359098185, 0.5550856819872789, 0.5467983404610478, 0.5410652082029204, 0.536255547908205, 0.6011915384635866, 0.6092812314836655, 0.6241321581033094, 0.6880744209704973, 2.915841858020288, 2.6264204515100484, 2.3560883070579246, 2.052603139729255, 2.421562802944245, 1.9231629319277221, 1.4565088666484294, 1.2845058483948424, 0.8068415180205192, 0.6295503966151642, 0.5969673730151487, 0.5830960955129549 ], "peak_layer": 21, "peak_normalized": 2.915841858020288, "population_cv": 0.7106082711854734, "rest_mean": 1.8080198363653228e-06, "spike_contrast": 3.4039814437617255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.154465972940671e-06, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.214581400039606e-06, 2.013782250287477e-06, 1.9291023818368558e-06, 1.906202328427753e-06, 1.695878950158658e-06, 1.6159390270331642e-06, 1.5588293535984121e-06, 1.5387429357360816e-06, 1.1616864412644645e-06, 1.1247253723922768e-06, 1.1173162874911213e-06, 1.1374067980796099e-06, 1.3805824892187957e-06, 1.3599706107925158e-06, 1.3457114391712821e-06, 1.3337490827325382e-06, 1.4952547644497827e-06, 1.5153750609897543e-06, 1.5523115735049942e-06, 1.7113457033701707e-06, 7.252142040670151e-06, 6.532306997542037e-06, 5.859949851583224e-06, 5.105136096972274e-06, 6.022794877935667e-06, 4.783198619406903e-06, 3.6225590065441793e-06, 3.1947613479132997e-06, 2.0067375317012193e-06, 1.5657875565011636e-06, 1.484748622715415e-06, 1.4502486465062248e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.464051883383945e-05, "first_to_last_ratio": 1.2085017018903634, "imbalance_abs_log_ratio": 0.18938132942715577, "last_quartile_mean": 2.866402155632386e-05, "mean": 3.421727353725146e-05, "normalized": [ 1.059950361009556, 1.2224231923999602, 1.1066505484293667, 1.0166204096806029, 1.0363170309532028, 0.9614969179125695, 0.8913049122896795, 0.8041913955656627, 0.6595104635459774, 0.6198402043713972, 0.5858567866753434, 0.5655998592669403, 0.7633893605247403, 0.7040501232645258, 0.6548491311439005, 0.6226536428847976, 0.7974262571555777, 0.7481683838617732, 0.7573132817097481, 0.8440618552058439, 3.1217920974447857, 2.4268766121530727, 1.8727793487823996, 1.45522837174708, 1.7605498171411489, 1.332061600542984, 0.9562588529424351, 0.830045802534728, 0.533682078716892, 0.4444300159412478, 0.423319578555294, 0.4213017056467665 ], "peak_layer": 21, "peak_normalized": 3.1217920974447857, "population_cv": 0.5796313993689626, "rest_mean": 2.7073180481912225e-05, "spike_contrast": 2.6888372455803045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.279537603608333e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.626861143857241e-05, 4.182798875262961e-05, 3.7866564525756985e-05, 3.478597864159383e-05, 3.5459943319438025e-05, 3.28998030454386e-05, 3.0498023988911882e-05, 2.751723695837427e-05, 2.2566649931832217e-05, 2.1209241822361946e-05, 2.0046421923325397e-05, 1.9353285097167827e-05, 2.6121102564502507e-05, 2.4090675651677884e-05, 2.2407151845982298e-05, 2.1305510017555207e-05, 2.7285752366879024e-05, 2.5600282242521644e-05, 2.5913195713656023e-05, 2.8881495381938294e-05, 0.00010681921412469819, 8.30411008791998e-05, 6.408140325220302e-05, 4.979394725523889e-05, 6.0241214669076726e-05, 4.557951615424827e-05, 3.2720570743549615e-05, 2.84019042737782e-05, 1.826114566938486e-05, 1.5207183423626702e-05, 1.4484841813100502e-05, 1.4415795703826007e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6040052716780338e-05, "first_to_last_ratio": 1.3601380691640932, "imbalance_abs_log_ratio": 0.3075862160391937, "last_quartile_mean": 1.9145153942190518e-05, "mean": 2.476174321941471e-05, "normalized": [ 1.1242415044070515, 1.2621466755459967, 1.1626111778912407, 1.0807522798319469, 1.0668131611583063, 0.9898612219145286, 0.9055963728518246, 0.8209725998576762, 0.7059309419194623, 0.6698532100359812, 0.6332217391849956, 0.605864404334209, 0.7980357739095889, 0.7372559068278083, 0.6856394720285257, 0.6565553942525388, 0.8578727127097449, 0.8064680621295278, 0.8069038983319957, 0.8750777012450611, 3.1375211320361442, 2.3884634889966896, 1.7586693778594196, 1.27827401077098, 1.559779974845964, 1.1976668654320728, 0.9315461470291763, 0.8389979859505794, 0.4605019751312927, 0.40913221789275306, 0.3951692615572719, 0.3926033521296457 ], "peak_layer": 21, "peak_normalized": 3.1375211320361442, "population_cv": 0.5588199044745877, "rest_mean": 2.0063699526805e-05, "spike_contrast": 2.4986009729926506, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0131179159507154e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.78381794487359e-05, 3.1252951885107905e-05, 2.8788279450964183e-05, 2.67613104369957e-05, 2.6416153559694067e-05, 2.451068939990364e-05, 2.2424144844990224e-05, 2.0328712707851082e-05, 1.7480080714449286e-05, 1.6586733181611635e-05, 1.5679674106650054e-05, 1.5002258805907331e-05, 1.9760756913456134e-05, 1.8255741451866925e-05, 1.697762854746543e-05, 1.6257456081802957e-05, 2.124242382706143e-05, 1.9969555069110356e-05, 1.9980347133241594e-05, 2.1668449335265905e-05, 7.769049261696637e-05, 5.914251960348338e-05, 4.3547719542402774e-05, 3.165229281876236e-05, 3.8622871215920895e-05, 2.96563193842303e-05, 2.3066706489771605e-05, 2.0775052689714357e-05, 1.140283166023437e-05, 1.013082692224998e-05, 9.785079782886896e-06, 9.721543392515741e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.001170565170469e-05, "first_to_last_ratio": 0.9626745767731016, "imbalance_abs_log_ratio": 0.038039850801722765, "last_quartile_mean": 9.350169602839742e-05, "mean": 9.76294164729552e-05, "normalized": [ 0.9359776868957951, 1.1142596124196091, 0.9861551159867753, 0.8961521822246734, 0.949684039356357, 0.8995149572091795, 0.846411402631371, 0.747630614092772, 0.5888085074137973, 0.5541267695979373, 0.5294131458395803, 0.5184084456985816, 0.7260183643779575, 0.6675529417749002, 0.6222711070136198, 0.5862023232639773, 0.7253041044725564, 0.6816202250286443, 0.6942528027900181, 0.79595263681847, 3.0863426882796725, 2.474889704855176, 2.017495070776267, 1.693791348848989, 2.023091547543398, 1.5725916380980733, 1.090726894893622, 0.9642815025941395, 0.6146236296489728, 0.48585096781736253, 0.45630461264981964, 0.45429340908793464 ], "peak_layer": 21, "peak_normalized": 3.0863426882796725, "population_cv": 0.6173495462344238, "rest_mean": 7.486509181304697e-05, "spike_contrast": 2.9460562232025813, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022055676963645966, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 9.137895540334284e-05, 0.00010878451575990766, 9.627774852560833e-05, 8.749081462156028e-05, 9.271709859604016e-05, 8.781912038102746e-05, 8.263465133495629e-05, 7.299074059119448e-05, 5.7485030993120745e-05, 5.409907316789031e-05, 5.168629650142975e-05, 5.06119140482042e-05, 7.088074926286936e-05, 6.517280417028815e-05, 6.075196506571956e-05, 5.723059075535275e-05, 7.081101648509502e-05, 6.654618482571095e-05, 6.777949602110311e-05, 7.770839147269726e-05, 0.0003013178356923163, 0.00024162203771993518, 0.0001969668664969504, 0.0001653638610150665, 0.00019751324725802988, 0.00015353120397776365, 0.00010648703027982265, 9.414224041393027e-05, 6.0005346313118935e-05, 4.743334648082964e-05, 4.4548753066919744e-05, 4.435240043676458e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08528144657611847, 0.3133305609226227, 0.3510810136795044, 0.36267346143722534, 0.1070452407002449, 0.1260577291250229, 0.14502130448818207, 0.17329032719135284, 0.061002109199762344, 0.0987050011754036, 0.13416865468025208, 0.17569874227046967, 0.06714949756860733, 0.09735818952322006, 0.13312570750713348, 0.17941679060459137, 0.10073095560073853, 0.13780468702316284, 0.17965665459632874, 0.24453195929527283, 0.04170985147356987, 0.07352267950773239, 0.10462787002325058, 0.14256280660629272, 0.030182205140590668, 0.06915032118558884, 0.10093247890472412, 0.1392749696969986, 0.030125634744763374, 0.08267941325902939, 0.15948866307735443, 0.2670789062976837 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1464836006780388e-05, "first_to_last_ratio": 0.8455234102400129, "imbalance_abs_log_ratio": 0.16779942298421527, "last_quartile_mean": 1.355945425984828e-05, "mean": 1.3058895916628899e-05, "normalized": [ 0.8914048885899613, 1.0775847922014457, 0.9537205719054163, 0.8574486865386339, 0.9048781790315702, 0.8425943237574262, 0.7839480584125434, 0.7118847066289794, 0.5788136508075115, 0.5307043127450849, 0.5023599445771596, 0.4844905191035793, 0.6659490427095206, 0.6212508418929359, 0.5836098650152179, 0.5530587758649329, 0.6705037956060123, 0.6390296236121042, 0.6539787044155942, 0.6823628402237571, 3.070552131396758, 2.5957343072679153, 2.1136920052807127, 1.723798777803649, 1.977372713282607, 1.75094231645673, 1.356176403898088, 0.9354566389044705, 0.8232143895880623, 0.5856320599681389, 0.4520406818345609, 0.4258114506789209 ], "peak_layer": 21, "peak_normalized": 3.070552131396758, "population_cv": 0.6442860381062553, "rest_mean": 9.924204715827256e-06, "spike_contrast": 3.0215245714485643, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.998622840095777e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.1640763659670483e-05, 1.4072067642700858e-05, 1.2454537682060618e-05, 1.119733315135818e-05, 1.1816709957201965e-05, 1.100335157389054e-05, 1.0237496098852716e-05, 9.296428288507741e-06, 7.558667221019277e-06, 6.9304123826441355e-06, 6.560266228916589e-06, 6.3269112615671474e-06, 8.696559234522283e-06, 8.112850082397927e-06, 7.621300483151572e-06, 7.22233698979835e-06, 8.75603927852353e-06, 8.345021342393011e-06, 8.540239832655061e-06, 8.91090530785732e-06, 4.0098020690493286e-05, 3.389742414583452e-05, 2.7602483896771446e-05, 2.2510908820549957e-05, 2.5822304451139644e-05, 2.2865373466629535e-05, 1.7710166503093205e-05, 1.2216030881972983e-05, 1.0750271030701697e-05, 7.6477081165649e-06, 5.9031522141594905e-06, 5.560627414524788e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.568271015159553e-05, "first_to_last_ratio": 0.8467191676221869, "imbalance_abs_log_ratio": 0.16638620055756023, "last_quartile_mean": 5.3952611324348254e-05, "mean": 5.204496915212076e-05, "normalized": [ 0.8954358371924881, 1.066255627359817, 0.9521818394578381, 0.863653814778354, 0.9057514302498563, 0.8375363338827018, 0.7860525947143857, 0.7151691796362102, 0.5850370033455559, 0.5346182010583443, 0.5035057922216662, 0.4847392766839134, 0.6665845013282112, 0.6225484700608501, 0.5847766932080282, 0.5518779375078734, 0.6703635418666527, 0.6385663510046552, 0.6481965678809122, 0.6867100259137117, 3.061813692446919, 2.6065600609918396, 2.1204217920704185, 1.7184136066569389, 1.970772681476859, 1.7371423758267366, 1.3426072722615594, 0.9360215708260244, 0.8279779050062409, 0.5898318056357381, 0.4571555590952484, 0.4317206583534514 ], "peak_layer": 21, "peak_normalized": 3.061813692446919, "population_cv": 0.6425899958232327, "rest_mean": 3.9558066755826415e-05, "spike_contrast": 3.0202244925053914, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011947424209211022, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.6602930524386466e-05, 5.549324123421684e-05, 4.955627446179278e-05, 4.494883614825085e-05, 4.7139805246843025e-05, 4.358955266070552e-05, 4.0910083043854684e-05, 3.722095789271407e-05, 3.044823279196862e-05, 2.7824187782243825e-05, 2.6204943424090743e-05, 2.5228240701835603e-05, 3.469236980890855e-05, 3.2400515920016915e-05, 3.0434684958891012e-05, 2.8722470233333297e-05, 3.488904985715635e-05, 3.3234166039619595e-05, 3.373537037987262e-05, 3.573980211513117e-05, 0.00015935199917294085, 0.0001356583379674703, 0.00011035728675778955, 8.943478314904496e-05, 0.00010256880341330543, 9.040952136274427e-05, 6.987595406826586e-05, 4.8715213779360056e-05, 4.309208452468738e-05, 3.0697778129251674e-05, 2.379264697083272e-05, 2.2468888346338645e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04051188472658396, "first_to_last_ratio": 0.846719171620572, "imbalance_abs_log_ratio": 0.16638619583535103, "last_quartile_mean": 0.047845715656876564, "mean": 0.04615399963222444, "normalized": [ 0.8954358499233408, 1.0662556745533454, 0.9521818348722626, 0.8636537656053828, 0.9057513866007281, 0.8375363761309916, 0.7860525635386049, 0.7151692172158937, 0.5850370145184794, 0.53461818747956, 0.5035057498056084, 0.48473925875942064, 0.666584502816651, 0.6225484458781422, 0.5847766703673021, 0.551877902951673, 0.670363548905047, 0.6385663306455772, 0.6481965620941408, 0.6867099814534051, 3.0618138122732117, 2.606560153011009, 2.1204217916267223, 1.7184136164632142, 1.9707726965272079, 1.7371424139057434, 1.3426073387034982, 0.9360215296259088, 0.8279778593866045, 0.5898318104039583, 0.45715553160267397, 0.43172062235469105 ], "peak_layer": 21, "peak_normalized": 3.0618138122732117, "population_cv": 0.6425900249685824, "rest_mean": 0.03508048918512133, "spike_contrast": 3.020224589442675, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10595095604658127, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04132794588804245, 0.04921196401119232, 0.0439470000565052, 0.03986107558012009, 0.04180404916405678, 0.03865565359592438, 0.03627946972846985, 0.033007919788360596, 0.027001798152923584, 0.02467476762831211, 0.023238804191350937, 0.02237265557050705, 0.030765540897846222, 0.02873310074210167, 0.026989782229065895, 0.02547137252986431, 0.03093995898962021, 0.029472390189766884, 0.02991686388850212, 0.03169441223144531, 0.14131495356559753, 0.12030317634344101, 0.09786594659090042, 0.07931166142225266, 0.09095904231071472, 0.08017607033252716, 0.06196669861674309, 0.04320113733410835, 0.038214489817619324, 0.027223097160458565, 0.021099556237459183, 0.019925633445382118 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8178741925112263e-06, "first_to_last_ratio": 0.5015983374707057, "imbalance_abs_log_ratio": 0.68995560412126, "last_quartile_mean": 3.6241631136135766e-06, "mean": 2.789416313930815e-06, "normalized": [ 0.8037911571270303, 0.7184217017994549, 0.6913125177981712, 0.6806867284307945, 0.605325927592527, 0.5818942838314957, 0.5658827679202331, 0.5663181288215361, 0.4369798716346615, 0.41689441779300684, 0.4078213694495809, 0.41540374891403675, 0.5056054983122913, 0.5034202979673874, 0.5059732849415142, 0.5032640783142611, 0.5409407291356678, 0.5584123873096231, 0.5703010167375573, 0.6038059140356934, 2.8715612743645713, 2.85957495161325, 2.4960596032794165, 2.196308283631167, 2.3935907596217825, 2.1483375605717328, 1.7201321959397329, 1.2992909597305788, 1.010652795641223, 0.7253891064801847, 0.563940476297644, 0.5327062049621921 ], "peak_layer": 21, "peak_normalized": 2.8715612743645713, "population_cv": 0.7722499515786573, "rest_mean": 1.981818833748413e-06, "spike_contrast": 3.608020362482595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.150442706915783e-06, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.2421081666834652e-06, 2.0039772152813384e-06, 1.9283584151708055e-06, 1.8987186649610521e-06, 1.688506017671898e-06, 1.623145408302662e-06, 1.5784826246090233e-06, 1.5796970274095656e-06, 1.2189187827971182e-06, 1.1628920901785023e-06, 1.137583581112267e-06, 1.1587339940888342e-06, 1.4103442254054244e-06, 1.4042487919141422e-06, 1.4113701354290242e-06, 1.4038130302651552e-06, 1.508908894720662e-06, 1.5576446230625152e-06, 1.5908069599390728e-06, 1.6842660670590703e-06, 8.009979865164496e-06, 7.97654502093792e-06, 6.962549377931282e-06, 6.126418156782165e-06, 6.676721113763051e-06, 5.992607839289121e-06, 4.798164809471928e-06, 3.6242633996153018e-06, 2.8191313958814135e-06, 2.023412207563524e-06, 1.573064764670562e-06, 1.4859393786537112e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.4892004123321385e-05, "first_to_last_ratio": 1.010196136459353, "imbalance_abs_log_ratio": 0.010144506514065088, "last_quartile_mean": 3.453983129020344e-05, "mean": 3.719143947478187e-05, "normalized": [ 0.9787509726025058, 1.1268290755091765, 1.0180314063323372, 0.9373900555049763, 0.9573129653471193, 0.8927633987058169, 0.8376905027658148, 0.756615575252793, 0.6421721653954596, 0.5898966583006985, 0.5545320603024807, 0.5324801984524263, 0.7191238348729887, 0.6685886503056807, 0.6276517726739922, 0.5907083001617862, 0.7404052849968692, 0.7009964656156153, 0.7076026287863885, 0.7352288733491559, 3.079466745153095, 2.56966898996169, 2.031234533994799, 1.57522845930543, 1.8426476034454873, 1.5558698594237106, 1.200138700827154, 0.8699239581339129, 0.6815952172432139, 0.48817942111819174, 0.40443641967892363, 0.3868392464803102 ], "peak_layer": 21, "peak_normalized": 3.079466745153095, "population_cv": 0.6121837817284164, "rest_mean": 2.879134468390408e-05, "spike_contrast": 2.8672488989954323, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.255195134552196e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.6401157558429986e-05, 4.190839536022395e-05, 3.786205343203619e-05, 3.486288551357575e-05, 3.5603847209131345e-05, 3.3203155908267945e-05, 3.11549156322144e-05, 2.813962237269152e-05, 2.388330722169485e-05, 2.193910586356651e-05, 2.06238455575658e-05, 1.9803705072263256e-05, 2.6745250579551794e-05, 2.4865774321369827e-05, 2.334327291464433e-05, 2.1969291992718354e-05, 2.7536738343769684e-05, 2.607106762297917e-05, 2.6316760340705514e-05, 2.7344220143277198e-05, 0.00011452980106696486, 9.556968871038407e-05, 7.554453623015434e-05, 5.85850139032118e-05, 6.853071681689471e-05, 5.786503970739432e-05, 4.463488585315645e-05, 3.23537242366001e-05, 2.5349507268401794e-05, 1.815609539335128e-05, 1.5041572623886168e-05, 1.4387108421942685e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6184012767771492e-05, "first_to_last_ratio": 1.1812258678199312, "imbalance_abs_log_ratio": 0.1665527702583449, "last_quartile_mean": 2.2166812868817942e-05, "mean": 2.6416474952384306e-05, "normalized": [ 1.0583395321083173, 1.18417524025499, 1.0887520391639638, 1.0135132254608268, 1.0023744568550392, 0.9356357677270415, 0.8663869238627898, 0.7804236609988654, 0.6948391179201333, 0.645410883875558, 0.609013089334734, 0.5813567671594309, 0.7649466228246298, 0.711734220686414, 0.6645112375219416, 0.6291608441906736, 0.8117379342921259, 0.7682870001881678, 0.7656778963007645, 0.7856293365916892, 3.109981047613895, 2.532125441581352, 1.911742908862577, 1.3712180563471432, 1.6504233650379605, 1.361126344660198, 1.0867081917704957, 0.8610084831897756, 0.5742988763844462, 0.4302061202657988, 0.3792452416796221, 0.3700101252886388 ], "peak_layer": 21, "peak_normalized": 3.109981047613895, "population_cv": 0.5830394716350688, "rest_mean": 2.0961481856863463e-05, "spike_contrast": 2.6655289950267567, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5873437668196856e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.7957599741057493e-05, 3.1281735573429614e-05, 2.8760990971932188e-05, 2.6773446734296158e-05, 2.6479199732420966e-05, 2.471619882271625e-05, 2.2886888473294675e-05, 2.061604209302459e-05, 1.8355200154474005e-05, 1.7049480447894894e-05, 1.6087979020085186e-05, 1.535739647806622e-05, 2.0207193301757798e-05, 1.880150921351742e-05, 1.7554044461576268e-05, 1.6620211681583896e-05, 2.144325480912812e-05, 2.029543429671321e-05, 2.0226510969223455e-05, 2.0753557691932656e-05, 8.215473644668236e-05, 6.688982830382884e-05, 5.050150866736658e-05, 3.62227474397514e-05, 4.3598367483355105e-05, 3.595615999074653e-05, 2.870699972845614e-05, 2.2744809029973112e-05, 1.5170951883192174e-05, 1.1364529200363904e-05, 1.0018322427640669e-05, 9.774363206815906e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.068014514923561e-05, "first_to_last_ratio": 0.7913424091198598, "imbalance_abs_log_ratio": 0.23402452356595546, "last_quartile_mean": 0.00011459027609816985, "mean": 0.00010754564084436424, "normalized": [ 0.8540109519123018, 1.0092398375236236, 0.8947367225413311, 0.8160956211381821, 0.862539829414268, 0.8245016378814637, 0.7800932919128317, 0.7042081007547356, 0.5666342926354664, 0.5256616841881218, 0.49571278437559146, 0.4792103383514405, 0.6722886591537548, 0.6295888625242309, 0.5934294393111791, 0.5594389550092885, 0.6645012760984752, 0.6282095204823469, 0.6412757892302924, 0.6736811938431846, 3.0068371075614557, 2.616557188534586, 2.1686834473810053, 1.8088340274566967, 2.0615653735311414, 1.8302172118360363, 1.4236627074872454, 0.9903997896635313, 0.8090570392266916, 0.5559293048344423, 0.43884492996422475, 0.4143530842408338 ], "peak_layer": 21, "peak_normalized": 3.0068371075614557, "population_cv": 0.6573647361404942, "rest_mean": 8.100784921057143e-05, "spike_contrast": 3.0966099965793172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00025084971566684543, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 9.184515511151403e-05, 0.00010853934509214014, 9.622503421269357e-05, 8.776752656558529e-05, 9.276239870814607e-05, 8.867155702318996e-05, 8.389563299715519e-05, 7.573451148346066e-05, 6.093904812587425e-05, 5.653262269333936e-05, 5.3311749070417136e-05, 5.1536982937250286e-05, 7.230171468108892e-05, 6.770953768864274e-05, 6.382074934663251e-05, 6.016522092977539e-05, 7.146421557990834e-05, 6.756119546480477e-05, 6.896641571074724e-05, 7.245147571666166e-05, 0.00032337222364731133, 0.00028139931964688003, 0.0002332324511371553, 0.0001945322146639228, 0.00022171236923895776, 0.00019683188293129206, 0.00015310871822293848, 0.00010651318007148802, 8.701055776327848e-05, 5.978777335258201e-05, 4.7195859224302694e-05, 4.456186798051931e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16101159155368805, 0.3334391713142395, 0.3570481836795807, 0.37708213925361633, 0.11129721254110336, 0.12943366169929504, 0.1512180119752884, 0.1945808082818985, 0.07549218833446503, 0.11765842139720917, 0.16134388744831085, 0.21493762731552124, 0.08329613506793976, 0.12312135100364685, 0.16809259355068207, 0.23101076483726501, 0.11538279056549072, 0.16158254444599152, 0.20080390572547913, 0.27125677466392517, 0.0543946772813797, 0.09335283935070038, 0.12897421419620514, 0.17114612460136414, 0.06173322722315788, 0.09693212062120438, 0.13771499693393707, 0.18387214839458466, 0.07983565330505371, 0.15526357293128967, 0.26127612590789795, 0.39550676941871643 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1373772622391698e-05, "first_to_last_ratio": 1.0318485034659548, "imbalance_abs_log_ratio": 0.03135185731624994, "last_quartile_mean": 1.1022715625585988e-05, "mean": 1.1832133452571725e-05, "normalized": [ 0.9800044623886516, 1.1858619463319722, 1.052769444074511, 0.9435868051213633, 0.9942662730454839, 0.9213689272443336, 0.8458140758926892, 0.7664188972680795, 0.6009172873544569, 0.5652322062780974, 0.5379371038219452, 0.5220047919249511, 0.7166699752090641, 0.6623360432121947, 0.616224251829878, 0.5903966461838739, 0.7324373700274538, 0.6882103836910076, 0.7092295284797906, 0.8089552049077647, 3.1154122124911208, 2.432133583525468, 1.964217211992137, 1.5948629117896413, 1.9522993684268721, 1.5038120812358047, 1.036848162520458, 0.8807155414212365, 0.6474761343085111, 0.5016554233955102, 0.4711561792646531, 0.4587695653410252 ], "peak_layer": 21, "peak_normalized": 3.1154122124911208, "population_cv": 0.6032722735217898, "rest_mean": 9.176947801110887e-06, "spike_contrast": 2.851725490613808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.617013597046025e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1595543583098333e-05, 1.4031276805326343e-05, 1.245650855707936e-05, 1.1164645002281759e-05, 1.1764291230065282e-05, 1.0901760106207803e-05, 1.0007785022025928e-05, 9.068370673048776e-06, 7.110133537935326e-06, 6.687902896373998e-06, 6.364943601511186e-06, 6.176430360937957e-06, 8.479734788124915e-06, 7.836848453735001e-06, 7.2912475843622815e-06, 6.985651907598367e-06, 8.66629670781549e-06, 8.142997103277594e-06, 8.391698429477401e-06, 9.571665941621177e-06, 3.686197305796668e-05, 2.8777329134754837e-05, 2.324088018212933e-05, 1.8870630810852163e-05, 2.3099866666598246e-05, 1.7793305232771672e-05, 1.2268125828995835e-05, 1.0420743819850031e-05, 7.661024028493557e-06, 5.935653916822048e-06, 5.574782790063182e-06, 5.428222721093334e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.529294619715074e-05, "first_to_last_ratio": 1.0314184626645753, "imbalance_abs_log_ratio": 0.03093500306067748, "last_quartile_mean": 4.391325910546584e-05, "mean": 4.717383586694268e-05, "normalized": [ 0.9832121149293677, 1.1731445898725168, 1.0505704052806353, 0.9501685038967237, 0.9944626347672784, 0.9147662213723499, 0.8464650022543867, 0.7682388544141363, 0.6067490194436969, 0.5681705700165066, 0.53864641920515, 0.5211305553096096, 0.7167263481864653, 0.6632364446817158, 0.6172546895643319, 0.588531029519173, 0.7318735922946267, 0.6893643707481226, 0.7026758072754231, 0.8079900073900919, 3.115049509935762, 2.4538624912763685, 1.9660898437352061, 1.5845676157597377, 1.937541165727475, 1.4892951790023068, 1.0350080319051866, 0.8840334683174624, 0.6516976556256888, 0.50650640646803, 0.47764736477161285, 0.4653240870528551 ], "peak_layer": 21, "peak_normalized": 3.115049509935762, "population_cv": 0.6027314599912318, "rest_mean": 3.6590978736596835e-05, "spike_contrast": 2.8510104941924466, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010432126437081024, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.638188693206757e-05, 5.534173033083789e-05, 4.9559435865376145e-05, 4.482309304876253e-05, 4.6912617108318955e-05, 4.315303158364259e-05, 3.9931001083459705e-05, 3.624077362474054e-05, 2.8622678655665368e-05, 2.680278521438595e-05, 2.541001776990015e-05, 2.4583727281424217e-05, 3.3810731110861525e-05, 3.128740718238987e-05, 2.9118271413608454e-05, 2.7763266189140268e-05, 3.4525284718256444e-05, 3.251996167819016e-05, 3.314791320008226e-05, 3.811598799075e-05, 0.00014694883429910988, 0.00011575810640351847, 9.27479995880276e-05, 7.475013262592256e-05, 9.14012489374727e-05, 7.025576633168384e-05, 4.882529901806265e-05, 4.170324973529205e-05, 3.074307824135758e-05, 2.38938500842778e-05, 2.2532458388013765e-05, 2.1951122107566334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.040166237857192755, "first_to_last_ratio": 1.0314184457192797, "imbalance_abs_log_ratio": 0.030934986631559365, "last_quartile_mean": 0.038942718179896474, "mean": 0.0418342292541638, "normalized": [ 0.9832121171072725, 1.1731446022953238, 1.0505704535174525, 0.9501684905945749, 0.9944626387492155, 0.9147662270065798, 0.8464650412985084, 0.7682388261849044, 0.6067490356906232, 0.5681705774712352, 0.5386464289336567, 0.5211305632866429, 0.7167263331576038, 0.6632364212118128, 0.6172547081682093, 0.5885310198061827, 0.7318735882573365, 0.6893643353923572, 0.7026757592562258, 0.8079899688020947, 3.1150494118332737, 2.453862539114119, 1.9660897667076618, 1.5845675971028321, 1.9375412374473495, 1.4892952935873054, 1.0350080081219681, 0.8840335073818288, 0.6516976484197683, 0.5065064256973429, 0.4776473385482515, 0.46532408985048596 ], "peak_layer": 21, "peak_normalized": 3.1150494118332737, "population_cv": 0.6027314563638121, "rest_mean": 0.03244924586680201, "spike_contrast": 2.851010464947826, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09251313954591751, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.041131921112537384, 0.0490776002407074, 0.043949805200099945, 0.039749566465616226, 0.04160257801413536, 0.03826854005455971, 0.03541121259331703, 0.032138679176568985, 0.02538287825882435, 0.02376897819340229, 0.022533858194947243, 0.021801095455884933, 0.02998369373381138, 0.027745984494686127, 0.02582237496972084, 0.024620741605758667, 0.0306173674762249, 0.028839025646448135, 0.029395898804068565, 0.03380163758993149, 0.13031569123268127, 0.10265544801950455, 0.08224985003471375, 0.06628916412591934, 0.08105554431676865, 0.06230352073907852, 0.043298762291669846, 0.036982860416173935, 0.02726326882839203, 0.021189305931329727, 0.019982008263468742, 0.019466474652290344 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8091637201678168e-06, "first_to_last_ratio": 0.5997838633923696, "imbalance_abs_log_ratio": 0.511185916342951, "last_quartile_mean": 3.0163594431087404e-06, "mean": 2.4871596693287756e-06, "normalized": [ 0.8903650291088842, 0.8096671832364504, 0.7756357252979299, 0.7664205976400669, 0.6819345308185697, 0.6497344242040057, 0.6267720257428505, 0.6186826317778965, 0.4671058445444584, 0.45225062104576175, 0.4492165155127133, 0.4573049495780591, 0.555081192151318, 0.5467853740610928, 0.5411086649066317, 0.5362598920190076, 0.601197329062199, 0.609347333830986, 0.6241580819811688, 0.6880678322200046, 2.9157464374333077, 2.626463595143142, 2.355941396471949, 2.0525708846936164, 2.4216180933419804, 1.9231831825131547, 1.4564932487560838, 1.2844496192048258, 0.8068213094456221, 0.6295223209584038, 0.59698327836313, 0.5831108549347289 ], "peak_layer": 21, "peak_normalized": 2.9157464374333077, "population_cv": 0.7105915967606766, "rest_mean": 1.8080416052894347e-06, "spike_contrast": 3.4039024307496555, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.154397215141216e-06, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.214479991380358e-06, 2.013771563724731e-06, 1.9291298940515844e-06, 1.9062104001932312e-06, 1.6960800621745875e-06, 1.6159932556547574e-06, 1.558882104291115e-06, 1.5387624898721697e-06, 1.1617668178587337e-06, 1.1248195050939103e-06, 1.1172732001796248e-06, 1.137390427174978e-06, 1.3805755543216947e-06, 1.3599425301435986e-06, 1.3458236480801133e-06, 1.3337639757082798e-06, 1.4952737501516822e-06, 1.5155441133174463e-06, 1.5523808087891666e-06, 1.711334562060074e-06, 7.251926945173182e-06, 6.532434326800285e-06, 5.8596024246071465e-06, 5.105071522848448e-06, 6.022950856277021e-06, 4.78326364827808e-06, 3.622531266955775e-06, 3.194631290170946e-06, 2.0066934212081833e-06, 1.5657225276299869e-06, 1.484792733208451e-06, 1.45028980114148e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.464059318503132e-05, "first_to_last_ratio": 1.2085006098527633, "imbalance_abs_log_ratio": 0.1893804257974035, "last_quartile_mean": 2.8664108981502068e-05, "mean": 3.421737719122575e-05, "normalized": [ 1.0599522534665642, 1.2224211904533802, 1.1066526183773722, 1.0166167984558396, 1.0363163370114612, 0.9614978327769692, 0.8912999264066335, 0.8041906605597857, 0.6595162801975746, 0.6198396025381588, 0.5858589457780455, 0.5655951157979022, 0.7633864100877904, 0.7040498510939207, 0.65484714742333, 0.6226490455428274, 0.7974248515656103, 0.7481679248848441, 0.7573152935416273, 0.844065677484853, 3.1218032666740525, 2.4268896738358103, 1.8727607046170862, 1.4552330006230305, 1.7605509694422596, 1.3320662835665362, 0.9562601026332771, 0.8300407364235332, 0.5336839174330044, 0.4444323110860877, 0.4233154787321452, 0.421299791488687 ], "peak_layer": 21, "peak_normalized": 3.1218032666740525, "population_cv": 0.5796332934591588, "rest_mean": 2.7073248087685056e-05, "spike_contrast": 2.6888415499527167, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.279567435034551e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.626878606155515e-05, 4.182804696029052e-05, 3.786675006267615e-05, 3.478596045169979e-05, 3.546002699295059e-05, 3.289993401267566e-05, 3.049794577236753e-05, 2.7517295166035183e-05, 2.2566917323274538e-05, 2.120928547810763e-05, 2.004655652854126e-05, 1.9353181414771825e-05, 2.6121080736629665e-05, 2.4090739316307008e-05, 2.2407151845982298e-05, 2.1305417249095626e-05, 2.728578692767769e-05, 2.5600344088161364e-05, 2.591334305179771e-05, 2.888171366066672e-05, 0.00010681991989258677, 8.304179937113076e-05, 6.408095941878855e-05, 4.9794256483437493e-05, 6.024143658578396e-05, 4.557981446851045e-05, 3.272071262472309e-05, 2.840181696228683e-05, 1.826126390369609e-05, 1.5207308024400845e-05, 1.4484745406662114e-05, 1.4415773875953164e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6039962676804862e-05, "first_to_last_ratio": 1.3600902622782072, "imbalance_abs_log_ratio": 0.3075510668678258, "last_quartile_mean": 1.914576068884344e-05, "mean": 2.476154466535263e-05, "normalized": [ 1.123958441324931, 1.2626354632798173, 1.1622790571900958, 1.0807265666022514, 1.0670121245383055, 0.9899364488555507, 0.9055408259899391, 0.8209444362453999, 0.7058751897560187, 0.6697662418103106, 0.6333373009906147, 0.6058288961358885, 0.7980579670309706, 0.7373130204202144, 0.6858704929138739, 0.6566823825816065, 0.8578675442054349, 0.8063397293421843, 0.8071587377333899, 0.875107050101149, 3.1369738883948477, 2.3886515998281816, 1.7583385106457243, 1.2781546769066958, 1.5599158953991201, 1.1978164843461745, 0.9315129197863214, 0.8390098557963587, 0.46047650401210477, 0.40917670978266013, 0.3951608602894278, 0.39257417776443665 ], "peak_layer": 21, "peak_normalized": 3.1369738883948477, "population_cv": 0.558754889865697, "rest_mean": 2.0064156227885246e-05, "spike_contrast": 2.4983578506037145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.012744222767651e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.7830947146867402e-05, 3.126480442006141e-05, 2.87798247882165e-05, 2.676045914995484e-05, 2.6420868380228058e-05, 2.4512355594197288e-05, 2.2422589609050192e-05, 2.0327852325863205e-05, 1.7478560039307922e-05, 1.6584446711931378e-05, 1.5682409866712987e-05, 1.5001259271230083e-05, 1.9761147996177897e-05, 1.8257009287481196e-05, 1.6983212844934314e-05, 1.6260470147244632e-05, 2.1242125512799248e-05, 1.996621722355485e-05, 1.9986497136414982e-05, 2.1669002308044583e-05, 7.767631905153394e-05, 5.9146703279111534e-05, 4.353917756816372e-05, 3.164908412145451e-05, 3.862592711811885e-05, 2.965978637803346e-05, 2.306569876964204e-05, 2.0775180018972605e-05, 1.1402109521441162e-05, 1.013184737530537e-05, 9.784793292055838e-06, 9.720743037178181e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.000182035379112e-05, "first_to_last_ratio": 0.9625877483741323, "imbalance_abs_log_ratio": 0.03813004983371345, "last_quartile_mean": 9.349986066808924e-05, "mean": 9.762683816916251e-05, "normalized": [ 0.9359774389006617, 1.1139547805652443, 0.9858638933617162, 0.896277058777463, 0.9496551618529471, 0.8993751236921529, 0.8464771316171845, 0.7475897673874267, 0.588915541149348, 0.5540229040457487, 0.5295015812507878, 0.5183640792456576, 0.7262152137199526, 0.6676039603614504, 0.6223324443316169, 0.5861986137162843, 0.7254235746253427, 0.6817432367794596, 0.6942948378413707, 0.7958728955594262, 3.0862176056110457, 2.4751904263802063, 2.0171676620749306, 1.6939489172879052, 2.0229993487511715, 1.572447147387479, 1.0908649591739474, 0.9643206077313163, 0.6147279541208738, 0.4857837184299343, 0.45630950884426186, 0.45436290542568586 ], "peak_layer": 21, "peak_normalized": 3.0862176056110457, "population_cv": 0.617326253069026, "rest_mean": 7.486342710089284e-05, "spike_contrast": 2.946021394940767, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022054925793781876, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 9.137651795754209e-05, 0.00010875188309000805, 9.624677477404475e-05, 8.750069537200034e-05, 9.271183080272749e-05, 8.780314965406433e-05, 8.263888594228774e-05, 7.298482523765415e-05, 5.7493962231092155e-05, 5.408750439528376e-05, 5.169356518308632e-05, 5.060624607722275e-05, 7.089809514582157e-05, 6.51760637992993e-05, 6.0756348830182105e-05, 5.72287171962671e-05, 7.082080992404372e-05, 6.655643664998934e-05, 6.778180977562442e-05, 7.769855437800288e-05, 0.0003012976667378098, 0.00024164501519408077, 0.00019692970090545714, 0.00016537487681489438, 0.00019749903003685176, 0.00015351304318755865, 0.00010649769683368504, 9.414357191417366e-05, 6.00139464950189e-05, 4.74255284643732e-05, 4.454805457498878e-05, 4.4358013838063926e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059412047266960144, 0.08935670554637909, 0.17747226357460022, 0.17946027219295502, 0.17260213196277618, 0.12674333155155182, 0.12412907183170319, 0.11767174303531647, 0.13582420349121094, 0.10855290293693542, 0.10545514523983002, 0.11005345731973648, 0.13049271702766418, 0.09666567295789719, 0.10358717292547226, 0.11041814833879471, 0.13209649920463562, 0.10239993035793304, 0.11153820157051086, 0.11912636458873749, 0.127323716878891, 0.07468672841787338, 0.07975661009550095, 0.08746611326932907, 0.09389390796422958, 0.06575146317481995, 0.0681111291050911, 0.07568862289190292, 0.08424623310565948, 0.0719028189778328, 0.08439797908067703, 0.08961156010627747 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.588972493977053e-06, "first_to_last_ratio": 1.465723686175851, "imbalance_abs_log_ratio": 0.38234910423701196, "last_quartile_mean": 5.1776283385152055e-06, "mean": 8.926610817638903e-06, "normalized": [ 0.8001707694815087, 1.2301759322409134, 0.8021990606376289, 0.7996309276786352, 0.4189560575118035, 0.770181199703876, 0.9964730820564364, 0.983425792900568, 0.6138765442017692, 0.5043130598626571, 0.39700601441149197, 0.37391212513577055, 0.38881218172106297, 0.7038080142822467, 0.6284832461563316, 0.48736568495191596, 0.7748207724466236, 1.0896235823984757, 1.2330955711393075, 1.3976034073521157, 2.3882743999212943, 4.5696632038041, 2.815663441519544, 2.192292008694111, 1.06869084333539, 1.167599927936825, 0.6186038915435874, 0.511201557417111, 0.4679157405813128, 0.21996309432798952, 0.26258255269417496, 0.3236163119534213 ], "peak_layer": 22, "peak_normalized": 4.5696632038041, "population_cv": 0.8757958168839719, "rest_mean": 6.270255131244927e-06, "spike_contrast": 3.711321315811602, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3270931524166372e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 7.14281304681208e-06, 1.0981301784340758e-05, 7.160918812587624e-06, 7.137994089134736e-06, 3.739857675100211e-06, 6.875107828818727e-06, 8.895127393770963e-06, 8.778659321251325e-06, 5.479837000166299e-06, 4.50180641564657e-06, 3.5439181829133304e-06, 3.3377680210833205e-06, 3.4707750273810234e-06, 6.2826202338328585e-06, 5.610225343843922e-06, 4.350523795437766e-06, 6.916523489053361e-06, 9.726645657792687e-06, 1.1007364264514763e-05, 1.2475861694838386e-05, 2.1319196093827486e-05, 4.079160498804413e-05, 2.513433173589874e-05, 1.956973756023217e-05, 9.539787242829334e-06, 1.0422710147395264e-05, 5.52203619008651e-06, 4.5632973524334375e-06, 4.176901711616665e-06, 1.9635249373095576e-06, 2.3439722554030595e-06, 2.8887968710478162e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.9898558636887174e-05, "first_to_last_ratio": 1.4621096880940556, "imbalance_abs_log_ratio": 0.37988038456939854, "last_quartile_mean": 2.0448916302484577e-05, "mean": 3.561926479278554e-05, "normalized": [ 0.7085385219770569, 1.1946020815752356, 0.7754024698062456, 0.7950922364389753, 0.4162410374634061, 0.8241424849247687, 0.9939701500428968, 1.0071543674888268, 0.5806040892681411, 0.5114068129628899, 0.4225456612843993, 0.35601883603809836, 0.39596565343777096, 0.700750155357531, 0.608338530669077, 0.5057982656951122, 0.7834207937068051, 1.0599792777683275, 1.2280812674610306, 1.4767524648757513, 2.409304768407655, 4.508238395616571, 2.902287830939038, 2.2425871184827204, 1.0734427327031963, 1.1671335490266965, 0.5965647485206973, 0.46999739473988966, 0.46865462397650987, 0.22456393251833243, 0.25925607539704565, 0.3331636714293014 ], "peak_layer": 22, "peak_normalized": 4.508238395616571, "population_cv": 0.8793156123901573, "rest_mean": 2.4886176578109843e-05, "spike_contrast": 3.760237771291333, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.357794115203433e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.523762123018969e-05, 4.255084786564112e-05, 2.761926589300856e-05, 2.832060090440791e-05, 1.482619973103283e-05, 2.9355349397519603e-05, 3.540448597050272e-05, 3.587409810279496e-05, 2.0680690795416012e-05, 1.8215934687759727e-05, 1.5050765796331689e-05, 1.2681129192060325e-05, 1.4104005458648317e-05, 2.4960205337265506e-05, 2.1668571207555942e-05, 1.80161623575259e-05, 2.790487269521691e-05, 3.775568256969564e-05, 4.374335185275413e-05, 5.2600837079808116e-05, 8.581766451243311e-05, 0.00016058013716246933, 0.00010337735875509679, 7.987930439412594e-05, 3.8235240936046466e-05, 4.157243893132545e-05, 2.1249197743600234e-05, 1.674096165515948e-05, 1.6693133147782646e-05, 7.998802175279707e-06, 9.234510798705742e-06, 1.1867045031976886e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.026514341006986797, "first_to_last_ratio": 1.4621096829473716, "imbalance_abs_log_ratio": 0.379880381049359, "last_quartile_mean": 0.01813430368201807, "mean": 0.03158751998853404, "normalized": [ 0.7085385035567081, 1.1946021626253802, 0.7754024540661398, 0.7950922593029921, 0.4162410309209195, 0.8241424880807221, 0.9939701540557254, 1.0071544006568143, 0.5806040665606002, 0.5114068209416861, 0.4225456864143744, 0.3560188250781175, 0.39596564639005655, 0.7007501603721394, 0.6083384888465566, 0.5057982797995526, 0.7834208304641355, 1.059979249016567, 1.2280813415682141, 1.4767524934129506, 2.409304781935525, 4.508238208880784, 2.902287717411148, 2.2425871343428208, 1.0734427669067585, 1.167133566980318, 0.5965647509522538, 0.46999742239503073, 0.4686546086713296, 0.22456393938088035, 0.25925609979218267, 0.3331636602206165 ], "peak_layer": 22, "peak_normalized": 4.508238208880784, "population_cv": 0.8793155841158318, "rest_mean": 0.022069310372764314, "spike_contrast": 3.7602376563657023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08298585191369057, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.022380974143743515, 0.03773451969027519, 0.024493040516972542, 0.025114992633461952, 0.013148021884262562, 0.026032617315649986, 0.031397052109241486, 0.03181350976228714, 0.018339842557907104, 0.016154073178768158, 0.013347170315682888, 0.01124575175344944, 0.012507572770118713, 0.02213495969772339, 0.0192159041762352, 0.015976913273334503, 0.024746321141719818, 0.03348211571574211, 0.038792043924331665, 0.04664694890379906, 0.07610396295785904, 0.14240406453609467, 0.09167607128620148, 0.07083776593208313, 0.03390739485621452, 0.03686685487627983, 0.01884400099515915, 0.01484605297446251, 0.014803636819124222, 0.007093417923897505, 0.008189257234334946, 0.010523813776671886 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.343415339112198e-06, "first_to_last_ratio": 0.7161840718970601, "imbalance_abs_log_ratio": 0.33381806140671977, "last_quartile_mean": 3.272085251637691e-06, "mean": 2.6718059498165303e-06, "normalized": [ 3.352170512510883, 1.252135055595148, 0.26582849245093404, 0.3779079423767161, 0.26494682358528443, 0.3763245935115975, 0.4665297795225812, 0.6608799701010516, 0.3129183029559072, 0.27411212897478776, 0.2550957946560799, 0.3550642916181046, 0.2737856172573793, 0.47262961496261435, 0.48685762845780345, 0.5118283391293013, 0.3284988072562945, 0.6155436681481607, 0.5758322948420541, 0.7857703722429643, 1.46149784943323, 3.1539420937482103, 2.83862849215308, 2.483897584167724, 1.672423695519606, 1.9844503515036886, 1.6695842115910071, 1.4607872550915708, 1.1268636557445397, 0.7047297682619745, 0.577264376468574, 0.6012706361611467 ], "peak_layer": 1, "peak_normalized": 3.352170512510883, "population_cv": 0.8823003320995952, "rest_mean": 2.0176697064387014e-06, "spike_contrast": 3.074904502088927, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.204141664056806e-06, "top_five_layers": [ 1, 22, 23, 24, 26 ], "top_five_spike_overlap": 3 }, "values": [ 8.956349120126106e-06, 3.3454618915129686e-06, 7.102421477611642e-07, 1.0096966889250325e-06, 7.078864996401535e-07, 1.0054662880065734e-06, 1.2464770406950265e-06, 1.7657430362305604e-06, 8.360569836440845e-07, 7.323744171117141e-07, 6.815664619352901e-07, 9.486628869126434e-07, 7.315020411624573e-07, 1.2627746173166088e-06, 1.300789108427125e-06, 1.36750600177038e-06, 8.776850677350012e-07, 1.6446132349301479e-06, 1.5385121514555067e-06, 2.099425955748302e-06, 3.904838649759768e-06, 8.426721251453273e-06, 7.5842644946533255e-06, 6.636492344114231e-06, 4.468391580303432e-06, 5.3020662562630605e-06, 4.4608050302485935e-06, 3.902940079569817e-06, 3.0107610200502677e-06, 1.8829011878551682e-06, 1.5423383956658654e-06, 1.6064784631453222e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.9795748357864795e-05, "first_to_last_ratio": 1.5829781482320753, "imbalance_abs_log_ratio": 0.4593079767809046, "last_quartile_mean": 1.2505383210736909e-05, "mean": 2.345933930314459e-05, "normalized": [ 0.6329672888423138, 1.1769775978635326, 0.8122053642936898, 0.8103566266208889, 0.46133266366701603, 0.8656603467447103, 1.0301469021057994, 0.9610116498331323, 0.61408958600413, 0.5510707221672605, 0.4548082316962977, 0.3885266091083219, 0.4407469545604743, 0.7576792724096253, 0.6251552614433971, 0.5326228510318479, 0.8273444966654151, 1.114509292339196, 1.2367312992724153, 1.4148882628934067, 2.3782100012844283, 4.280445457492481, 3.0878808679778498, 2.2801019909661657, 1.0813850741694784, 1.1060747126282833, 0.5844412396792442, 0.4054991266359997, 0.4197995696214123, 0.21544640490341943, 0.19403464509867754, 0.2578496299796893 ], "peak_layer": 22, "peak_normalized": 4.280445457492481, "population_cv": 0.8646936695537649, "rest_mean": 1.6414566272470876e-05, "spike_contrast": 3.746740099488776, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.150111366878263e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.4848994396743365e-05, 2.7611116820480675e-05, 1.9053801224799827e-05, 1.9010431060451083e-05, 1.0822559488588013e-05, 2.0307819795561954e-05, 2.416656570858322e-05, 2.2544698367710225e-05, 1.4406135960598476e-05, 1.2927755051350687e-05, 1.0669500625226647e-05, 9.11457755137235e-06, 1.0339632353861816e-05, 1.7774655134417117e-05, 1.4665729395346716e-05, 1.2494980182964355e-05, 1.9408955267863348e-05, 2.6145651645492762e-05, 2.9012899176450446e-05, 3.319234383525327e-05, 5.5791235354263335e-05, 0.00010041642235592008, 7.2439645009581e-05, 5.34896862518508e-05, 2.5368579372297972e-05, 2.5947781978175044e-05, 1.371060534438584e-05, 9.512741598882712e-06, 9.84822054306278e-06, 5.05423031427199e-06, 4.551924575935118e-06, 6.0489819588838145e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.3026621559220075e-05, "first_to_last_ratio": 1.6815341811376112, "imbalance_abs_log_ratio": 0.5197065797570352, "last_quartile_mean": 7.74686694171578e-06, "mean": 1.525919709166601e-05, "normalized": [ 0.5774672585272599, 1.1989902313733303, 0.8415468606033967, 0.8210487753497788, 0.5264205831559432, 0.9142648476833113, 1.0733362714129566, 0.8764439105095523, 0.6569979339342769, 0.6167902521881127, 0.4964350034575567, 0.41696619601743, 0.504954663560933, 0.8317766688879529, 0.6542358689500138, 0.5551252336862755, 0.852113468285689, 1.140364193630489, 1.2382881917527824, 1.3141660941820785, 2.300727833390314, 4.0533116788495525, 3.2452755011871197, 2.231471710715914, 1.0423588969466941, 1.097025384808554, 0.5832116217690613, 0.3693409741830814, 0.3982639506890586, 0.21137018039235425, 0.16048903759429983, 0.19942072232487634 ], "peak_layer": 22, "peak_normalized": 4.0533116788495525, "population_cv": 0.8414682188802379, "rest_mean": 1.0809645952273664e-05, "spike_contrast": 3.6344181315319797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.9286773244384676e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.811686711851507e-06, 1.8295628251507878e-05, 1.2841329407820012e-05, 1.2528545084933285e-05, 8.032755431486294e-06, 1.3950947504781652e-05, 1.6378249711124226e-05, 1.3373830370255746e-05, 1.0025260962720495e-05, 9.411724022356793e-06, 7.575199560960755e-06, 6.362569365592208e-06, 7.705202733632177e-06, 1.2692244126810692e-05, 9.983114068745635e-06, 8.47076535137603e-06, 1.3002567357034422e-05, 1.7401041986886412e-05, 1.889528357423842e-05, 2.0053119442309253e-05, 3.510725946398452e-05, 6.185028178151697e-05, 4.952029848936945e-05, 3.4050466638291255e-05, 1.5905559848761186e-05, 1.6739726561354473e-05, 8.899341082724277e-06, 5.635846719087567e-06, 6.077188118069898e-06, 3.225339241907932e-06, 2.448933855703217e-06, 3.043000106117688e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.830217355651257e-05, "first_to_last_ratio": 1.5342432776570591, "imbalance_abs_log_ratio": 0.42803728076015524, "last_quartile_mean": 3.8000605513843766e-05, "mean": 7.032809671159157e-05, "normalized": [ 0.6819589535091998, 1.1712319271696168, 0.7869167841712673, 0.779993936454377, 0.40965657804504296, 0.8112730725890709, 0.9714853978881673, 1.0195039829060222, 0.5575391424076114, 0.48579339303929814, 0.4244288891192081, 0.3806909807485322, 0.3969007539297048, 0.6868963514684149, 0.5698714639132343, 0.5145191503639089, 0.8281057882616928, 1.0694789461698069, 1.2364533982124963, 1.5208781026521505, 2.5059295507917216, 4.611853308457518, 2.9277607037675337, 2.3282138476317833, 1.1223656127309714, 1.0704287879823866, 0.5800732884744664, 0.412100266213389, 0.4197925520111647, 0.21365215237778729, 0.20903494194495364, 0.2952179945975007 ], "peak_layer": 22, "peak_normalized": 4.611853308457518, "population_cv": 0.9090106345148538, "rest_mean": 4.819786850189471e-05, "spike_contrast": 3.9385834881161212, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001898313290439546, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 4.796087523573078e-05, 8.237051224568859e-05, 5.534235970117152e-05, 5.4855488997418433e-05, 2.881036743929144e-05, 5.705529110855423e-05, 6.832271901657805e-05, 7.169977470766753e-05, 3.921066672774032e-05, 3.4164924727519974e-05, 2.984927596116904e-05, 2.6773272111313418e-05, 2.7913274607271887e-05, 4.830811303691007e-05, 4.007797542726621e-05, 3.6185152566758916e-05, 5.82391039642971e-05, 7.521441875724122e-05, 8.695741416886449e-05, 0.00010696046228986233, 0.0001762372558005154, 0.00032434286549687386, 0.00020590383792296052, 0.00016373884864151478, 7.893383735790849e-05, 7.528121932409704e-05, 4.0795450331643224e-05, 2.8982227377127856e-05, 2.9523211196647026e-05, 1.5025749235064723e-05, 1.4701029613206629e-05, 2.0762119675055146e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.056020382791757584, 0.0742802768945694, 0.1285802572965622, 0.10598119348287582, 0.10133238881826401, 0.10985831171274185, 0.12611564993858337, 0.1125042662024498, 0.07848236709833145, 0.0899009183049202, 0.09790628403425217, 0.09713682532310486, 0.09147685021162033, 0.10080859810113907, 0.09896469116210938, 0.10007023066282272, 0.10974904894828796, 0.10291008651256561, 0.10674569010734558, 0.12148815393447876, 0.055056143552064896, 0.05981758236885071, 0.07101839780807495, 0.08339047431945801, 0.06143829599022865, 0.06018613278865814, 0.06816941499710083, 0.08121132105588913, 0.05916989967226982, 0.07458337396383286, 0.08333029597997665, 0.09104464948177338 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.630958900657788e-06, "first_to_last_ratio": 0.20421556767006768, "imbalance_abs_log_ratio": 1.5885791387918327, "last_quartile_mean": 1.778002990704408e-05, "mean": 1.0223539980813712e-05, "normalized": [ 0.8429681862296823, 0.35291594683302396, 0.22484209501334745, 0.36790271245902445, 0.21868611610463093, 0.18545641795638437, 0.23387248761479024, 0.41460977766546225, 0.4054488572719161, 0.3330083345972086, 0.3085788428775866, 0.3192217174198833, 0.3816324391793382, 0.4112131193630629, 0.4344687728023616, 0.46821459950254946, 0.37185344111151275, 0.4758162142971942, 0.41846445085124506, 0.6781708994429974, 2.4552099328163637, 2.8496156807280086, 2.603123697672816, 2.3316927886492507, 2.616234924154862, 2.65437990886044, 2.4006671523655148, 1.672837600494882, 1.585060117848551, 1.2276893902450918, 0.8940161400914827, 0.8621272374795348 ], "peak_layer": 22, "peak_normalized": 2.8496156807280086, "population_cv": 0.8996576825534632, "rest_mean": 7.248915060834541e-06, "spike_contrast": 3.626268804103626, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6286514548701233e-05, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 8.618118954473175e-06, 3.6080502923141466e-06, 2.298682147738873e-06, 3.7612680898746476e-06, 2.2357462512445636e-06, 1.8960211036755936e-06, 2.3910047275421675e-06, 4.238779638399137e-06, 4.145122602494666e-06, 3.4045240226987517e-06, 3.1547681373922387e-06, 3.263575990786194e-06, 3.9016344999254216e-06, 4.204053766443394e-06, 4.441808869160013e-06, 4.786810677614994e-06, 3.801658522206708e-06, 4.864526090386789e-06, 4.278188043826958e-06, 6.933307304279879e-06, 2.5100936909439042e-05, 2.9133159841876477e-05, 2.661313919816166e-05, 2.383815444773063e-05, 2.674718234629836e-05, 2.7137159122503363e-05, 2.4543316612835042e-05, 1.71023220900679e-05, 1.6204925486817956e-05, 1.2551331565191504e-05, 9.140009751718026e-06, 8.813992280920502e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.470679973181177e-05, "first_to_last_ratio": 0.2069929445185519, "imbalance_abs_log_ratio": 1.5750705707490962, "last_quartile_mean": 7.104976339178393e-05, "mean": 4.0840292868438155e-05, "normalized": [ 0.8622316727455452, 0.3538749471728574, 0.22575416255506364, 0.3708511987720451, 0.22255638964420116, 0.1850393452789397, 0.23623438814969275, 0.4242990843945505, 0.41091535564437, 0.3361491695289819, 0.31238992658037873, 0.3212003149521768, 0.3849834068983422, 0.4107834308653127, 0.4382023170687432, 0.4728175618985744, 0.3691518544033243, 0.4681213159064821, 0.4170992962583633, 0.6272493229024287, 2.4438193245013227, 2.8483751664462695, 2.6052355716893176, 2.3350839107599644, 2.618275725913621, 2.653345803911492, 2.3878946201876037, 1.6698710725380324, 1.59563349859018, 1.224673847138525, 0.9002294831629131, 0.8676575135403847 ], "peak_layer": 22, "peak_normalized": 2.8483751664462695, "population_cv": 0.8985289334793173, "rest_mean": 2.8965161365906373e-05, "spike_contrast": 3.6238708169484144, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010496600298210978, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 3.521379403537139e-05, 1.4452356481342576e-05, 9.219866115017794e-06, 1.5145671568461694e-05, 9.089268132811412e-06, 7.557061053375946e-06, 9.647881597629748e-06, 1.73284988704836e-05, 1.678190346865449e-05, 1.3728430531045888e-05, 1.2758096090692561e-05, 1.3117914932081476e-05, 1.572283508721739e-05, 1.6776515622041188e-05, 1.7896310964715667e-05, 1.9310007701278664e-05, 1.5076269846758805e-05, 1.9118211639579386e-05, 1.703445741441101e-05, 2.5617046048864722e-05, 9.980629693018273e-05, 0.00011632847599685192, 0.00010639858373906463, 9.536551078781486e-05, 0.00010693114745663479, 0.0001083634197129868, 9.752231562742963e-05, 6.819802365498617e-05, 6.516613939311355e-05, 5.0016038585454226e-05, 3.6765635741176084e-05, 3.5435386962490156e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.013042137026786804, "first_to_last_ratio": 0.20699294542918592, "imbalance_abs_log_ratio": 1.5750705663497482, "last_quartile_mean": 0.06300764018669724, "mean": 0.03621758034569211, "normalized": [ 0.8622316598933797, 0.3538749471508598, 0.22575414844580807, 0.37085122221171446, 0.22255637683921708, 0.1850393384194697, 0.236234387422702, 0.42429908068863703, 0.41091534861415707, 0.3361491670478898, 0.3123899060706743, 0.3212003125118064, 0.3849834050274649, 0.41078345814992373, 0.43820230021845763, 0.4728175684224875, 0.36915184353764025, 0.46812130102476074, 0.41709928593336826, 0.6272492932959962, 2.4438193692237777, 2.8483752780000278, 2.605235605762955, 2.3350840258688303, 2.6182758157645756, 2.6533456676353473, 2.3878945209130196, 1.6698711077439734, 1.5956335166208298, 1.2246738042807082, 0.9002294337837535, 0.8676575034757876 ], "peak_layer": 22, "peak_normalized": 2.8483752780000278, "population_cv": 0.8985289457588168, "rest_mean": 0.025686594381652498, "spike_contrast": 3.623871003234085, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09308490455150605, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 0.031227944418787956, 0.012816494330763817, 0.008176269009709358, 0.013431333936750889, 0.008060453459620476, 0.006701677106320858, 0.008555837906897068, 0.015367086045444012, 0.014882359653711319, 0.012174509465694427, 0.01131400652229786, 0.011633098125457764, 0.01394316740334034, 0.014877582900226116, 0.015870627015829086, 0.01712430827319622, 0.01336978655308485, 0.016954220831394196, 0.015106326900422573, 0.02271745167672634, 0.0885092243552208, 0.10316126048564911, 0.09435532987117767, 0.08457109332084656, 0.09482761472463608, 0.09609775990247726, 0.08648376166820526, 0.06047869101166725, 0.057789985090494156, 0.044354721903800964, 0.03260413184762001, 0.03142445534467697 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 8.568965839117482e-07, "first_to_last_ratio": 0.18051220552416403, "imbalance_abs_log_ratio": 1.7119568828628857, "last_quartile_mean": 4.747028498286454e-06, "mean": 2.6146394631609837e-06, "normalized": [ 0.8589787963589357, 0.3082289930915967, 0.22695141403248437, 0.38502719312209793, 0.20717193772164363, 0.14969905615852777, 0.1917600510841699, 0.2940249836979971, 0.3501028028074662, 0.30509132547078094, 0.2840337285464057, 0.28520738633381276, 0.33814434197742116, 0.362621645282935, 0.4284643797973123, 0.4847838221845296, 0.3528575777552836, 0.44423254880618596, 0.4021648360838168, 0.39838852122464097, 2.445175567674409, 2.9178538331950583, 2.621993890930227, 2.432580720612022, 2.6636416217847234, 2.7243600687846947, 2.580677484419162, 1.8055086156001192, 1.6378764119490026, 1.2592445178301535, 0.939779150965119, 0.9133727747172655 ], "peak_layer": 22, "peak_normalized": 2.9178538331950583, "population_cv": 0.9456027936919144, "rest_mean": 1.8320637688399681e-06, "spike_contrast": 3.7337937296941295, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.840548212494468e-06, "top_five_layers": [ 22, 26, 25, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 2.2459198589785956e-06, 8.059076890276629e-07, 5.933961233495211e-07, 1.0067072935271426e-06, 5.416799240265391e-07, 3.9140905983003904e-07, 5.013833970224368e-07, 7.687693255320482e-07, 9.153926043836691e-07, 7.977038194439956e-07, 7.426457955261867e-07, 7.457144874933874e-07, 8.841255407787685e-07, 9.481248639531259e-07, 1.1202798759768484e-06, 1.2675349125856883e-06, 9.225953476743598e-07, 1.1615079529292416e-06, 1.0515160511204158e-06, 1.0416423492642934e-06, 6.39325253359857e-06, 7.629135780007346e-06, 6.855568699393189e-06, 6.360321549436776e-06, 6.964462500036461e-06, 7.123219347704435e-06, 6.747541192453355e-06, 4.7207540774252266e-06, 4.2824563024623785e-06, 3.2924704100878444e-06, 2.4571836547693238e-06, 2.388140501352609e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 9.912464463468496e-06, "first_to_last_ratio": 0.22715332429513255, "imbalance_abs_log_ratio": 1.4821300520824365, "last_quartile_mean": 4.363776974969369e-05, "mean": 2.6559063869058264e-05, "normalized": [ 0.9366358613935047, 0.3792603305202499, 0.18134819814305192, 0.34760749803128715, 0.23747281376634713, 0.19753839084684555, 0.24598063870488243, 0.4599435726461178, 0.4623870697047853, 0.36990754631923695, 0.34903068824582323, 0.37085556359303906, 0.4380455173785828, 0.45529019266842574, 0.48514444968944176, 0.5201735235841253, 0.4152101632813542, 0.5283175891854636, 0.46450504110947355, 0.6293184811026621, 2.502663692008064, 2.9461029153808944, 2.6595225397265994, 2.273367886792679, 2.65101572499286, 2.525147631813147, 2.234627662919172, 1.5898700335733806, 1.426452730852729, 1.112160691183748, 0.8183556047810898, 0.7867397560609359 ], "peak_layer": 22, "peak_normalized": 2.9461029153808944, "population_cv": 0.880175635609515, "rest_mean": 1.8657572430432385e-05, "spike_contrast": 3.7104032636486823, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.9227117637638e-05, "top_five_layers": [ 22, 23, 25, 26, 21 ], "top_five_spike_overlap": 4 }, "values": [ 2.4876171664800495e-05, 1.0072799341287464e-05, 4.816438377019949e-06, 9.2321297415765e-06, 6.307055627985392e-06, 5.246434739092365e-06, 6.533015493914718e-06, 1.2215670722071081e-05, 1.2280567716516089e-05, 9.824398148339242e-06, 9.269928341382183e-06, 9.849576599663123e-06, 1.1634078873612452e-05, 1.2092081306036562e-05, 1.2884982425021008e-05, 1.381532183586387e-05, 1.1027593245671596e-05, 1.4031620594323613e-05, 1.2336819054326043e-05, 1.6714109733584337e-05, 6.646840483881533e-05, 7.824573549441993e-05, 7.063442899379879e-05, 6.037852290319279e-05, 7.040849595796317e-05, 6.70655572321266e-05, 5.934961882303469e-05, 4.222545976517722e-05, 3.788524918491021e-05, 2.953794682980515e-05, 2.1734758774982765e-05, 2.0895071429549716e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 6.6299761840582505e-06, "first_to_last_ratio": 0.25432294789989096, "imbalance_abs_log_ratio": 1.369150371144901, "last_quartile_mean": 2.6069122895933106e-05, "mean": 1.693132426083821e-05, "normalized": [ 1.009812755573757, 0.39968507734198944, 0.16094253648199763, 0.32854895069647194, 0.25477694605768825, 0.2155789505292242, 0.25860705350981517, 0.5046916184590391, 0.515329188356828, 0.40705935830374973, 0.38337515923971865, 0.42269040487394594, 0.4945745095736154, 0.5045501286967757, 0.5247552857381682, 0.5730846754557868, 0.46495437450577576, 0.5946150769636939, 0.5125942030699491, 0.6193454891464502, 2.628840032502442, 3.0610378618268497, 2.7106344368298605, 2.1323334410119363, 2.565140847337925, 2.330500972416668, 2.0157738095858844, 1.503546921244241, 1.3029013314823592, 1.0664512040978233, 0.7882888849242707, 0.7449785141652996 ], "peak_layer": 22, "peak_normalized": 3.0610378618268497, "population_cv": 0.8541047749122238, "rest_mean": 1.1853189545473907e-05, "spike_contrast": 3.7418832756911033, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.435325172380544e-05, "top_five_layers": [ 22, 23, 21, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.7097467207349837e-05, 6.767197646695422e-06, 2.724970272538485e-06, 5.562768819800112e-06, 4.313711087888805e-06, 3.650037115221494e-06, 4.3785598791146185e-06, 8.54509744385723e-06, 8.725205589144025e-06, 6.892053988849511e-06, 6.4910491346381605e-06, 7.156708306865767e-06, 8.373801392735913e-06, 8.54270183481276e-06, 8.884801900421735e-06, 9.703082469059154e-06, 7.872293281252496e-06, 1.0067620678455569e-05, 8.678898666403256e-06, 1.0486339306226e-05, 4.45097430201713e-05, 5.182742461329326e-05, 4.589463060256094e-05, 3.610322892200202e-05, 4.34312314609997e-05, 3.945846765418537e-05, 3.412972000660375e-05, 2.5457040464971215e-05, 2.2059844923205674e-05, 1.8056431144941598e-05, 1.3346774721867405e-05, 1.2613472790690139e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.929049958311225e-05, "first_to_last_ratio": 0.19264517754015853, "imbalance_abs_log_ratio": 1.6469052404570816, "last_quartile_mean": 0.00015204377268673852, "mean": 8.56582414883178e-05, "normalized": [ 0.8435321808032064, 0.34370726972583865, 0.1808664069876229, 0.33993318208269363, 0.2101403998193939, 0.17675988933980996, 0.23091552849720237, 0.4097137534640246, 0.40325567762898656, 0.3332485242329118, 0.315649429844806, 0.31114263500046885, 0.3744856227345994, 0.39208745649289284, 0.44367077395553656, 0.4702768538142674, 0.35548322757045653, 0.45205464826790226, 0.41818583974351736, 0.6083319800365703, 2.3949709614831773, 2.7856906015999057, 2.6171687947428244, 2.3886915605377412, 2.7341447434319206, 2.7222211356577324, 2.584598405216518, 1.8271047030330585, 1.5687480327138839, 1.1190385826225775, 0.8424439925404574, 0.8017372063774949 ], "peak_layer": 22, "peak_normalized": 2.7856906015999057, "population_cv": 0.9245587442879203, "rest_mean": 6.052970898222325e-05, "spike_contrast": 3.6569202255083786, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022135231702122838, "top_five_layers": [ 22, 25, 26, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 7.22554832464084e-05, 2.944136031146627e-05, 1.5492698366870172e-05, 2.911807860073168e-05, 1.8000257114181295e-05, 1.5140941286517773e-05, 1.977981810341589e-05, 3.509535963530652e-05, 3.4542172215878963e-05, 2.8545482564368285e-05, 2.7037975087296218e-05, 2.665193096618168e-05, 3.207777990610339e-05, 3.358552203280851e-05, 3.800405829679221e-05, 4.028308831038885e-05, 3.04500681522768e-05, 3.8722206227248535e-05, 3.582106364774518e-05, 5.2108647651039064e-05, 0.00020514900097623467, 0.000238617358263582, 0.0002241820766357705, 0.00020461111853364855, 0.00023420203069690615, 0.00023318067542277277, 0.00022139215434435755, 0.0001565065758768469, 0.00013437619782052934, 9.58548771450296e-05, 7.21622709534131e-05, 6.867539923405275e-05 ] } } } } } }, "step": 4000 }, { "forward_identity_gate": { "comparisons": { "detached_learned": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_value_backward": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "modes": { "detached_learned": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4657019391961512e-05, "first_to_last_ratio": 0.6094218601945491, "imbalance_abs_log_ratio": 0.4952445414057865, "last_quartile_mean": 2.4050695174082648e-05, "mean": 2.0123495630741672e-05, "normalized": [ 0.6757922451357102, 0.7862872769192628, 0.6956720089134658, 0.6457028717597475, 0.8486081172555344, 0.7866590564328257, 0.7355608412572463, 0.6525459469054529, 0.5676581895263636, 0.5065196225058339, 0.47654698952417385, 0.4567643060278592, 0.6388028952758977, 0.6012584522984605, 0.5677051478189228, 0.5320958926863361, 0.6601484899512206, 0.6199678744151763, 0.6252065487643861, 0.6703348689792878, 3.1476439082057013, 2.6843722856032843, 2.147025515739066, 1.709881143629556, 2.310023308278431, 2.0253173950794507, 1.5260326037153789, 0.9852764539113186, 1.0403760218043532, 0.7424001651630263, 0.4909889920627068, 0.44082456445456286 ], "peak_layer": 21, "peak_normalized": 3.1476439082057013, "population_cv": 0.7023423019155828, "rest_mean": 1.4907078501117346e-05, "spike_contrast": 3.2395447657361793, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.829214813071303e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.359930229227757e-05, 1.5822848581592552e-05, 1.3999352631799411e-05, 1.299379891861463e-05, 1.7076961739803664e-05, 1.5830330085009336e-05, 1.4802055375184864e-05, 1.3131505511410069e-05, 1.1423267096688505e-05, 1.0192945410381071e-05, 9.58979126153281e-06, 9.191694516630378e-06, 1.2854947271989658e-05, 1.2099421837774571e-05, 1.1424212061683647e-05, 1.0707629371609073e-05, 1.32844952531741e-05, 1.2475920811994001e-05, 1.2581341252371203e-05, 1.348948080703849e-05, 6.334159843390808e-05, 5.401895396062173e-05, 4.3205658585065976e-05, 3.4408785722916946e-05, 4.648574395105243e-05, 4.075646575074643e-05, 3.0709110433235765e-05, 1.9827206415357068e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.829688234371133e-05, "first_to_last_ratio": 0.6072503732166264, "imbalance_abs_log_ratio": 0.4988140964987654, "last_quartile_mean": 9.600139401300112e-05, "mean": 8.040016996346822e-05, "normalized": [ 0.6783743745296411, 0.7863508052165757, 0.7000215696280339, 0.647124428880743, 0.8353484843612504, 0.7743411566859596, 0.7293747929726763, 0.6497369355766927, 0.5707307458024328, 0.510545581500116, 0.4799754930157162, 0.4590012320770852, 0.6477982227600472, 0.6029757489491335, 0.5684984161378385, 0.5312301622013348, 0.6575876681964574, 0.6204772859169876, 0.6292313575338936, 0.6709435017853759, 3.152621969125565, 2.693161601564409, 2.150493956812164, 1.7016971884685128, 2.3039790532500866, 2.0188035652467615, 1.519776732418517, 0.9811964658615112, 1.039948884922119, 0.7451655332994308, 0.496455864160715, 0.4470312211422162 ], "peak_layer": 21, "peak_normalized": 3.152621969125565, "population_cv": 0.7020159094649734, "rest_mean": 5.954986355155362e-05, "spike_contrast": 3.24084411076324, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019299182458780705, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.454141501104459e-05, 6.322273839032277e-05, 5.6281853176187724e-05, 5.2028914069524035e-05, 6.716216012137011e-05, 6.225716060725972e-05, 5.864185732207261e-05, 5.223896005190909e-05, 4.588684896589257e-05, 4.104795152670704e-05, 3.8590111216763034e-05, 3.690377707243897e-05, 5.208308721194044e-05, 4.847935269935988e-05, 4.57073692814447e-05, 4.27109953307081e-05, 5.287016028887592e-05, 4.988647924619727e-05, 5.059030809206888e-05, 5.394397157942876e-05, 0.0002534713421482593, 0.00021653065050486475, 0.00017290007963310927, 0.00013681674317922443, 0.00018524030747357756, 0.00016231214976869524, 0.0001221903075929731, 7.888836262281984e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05169825907796621, "first_to_last_ratio": 0.607250394007124, "imbalance_abs_log_ratio": 0.4988140622616565, "last_quartile_mean": 0.08513499470427632, "mean": 0.07129967457149178, "normalized": [ 0.6783743842436116, 0.7863508228047997, 0.700021554566655, 0.6471244375634602, 0.8353485410067275, 0.7743411217427667, 0.7293748339206622, 0.6497369595226863, 0.5707307154188002, 0.5105455806753446, 0.47997548951564134, 0.4590012256825957, 0.6477982322753953, 0.6029757319104283, 0.5684984046970705, 0.5312301796601451, 0.6575876919843755, 0.6204773092105361, 0.6292313633824181, 0.6709435195164998, 3.152621923151655, 2.693161643472544, 2.150493986671358, 1.7016971770874902, 2.30397915541657, 2.0188034853391237, 1.5197766802362542, 0.9811964117969386, 1.0399488445555949, 0.7451654867762114, 0.49645587282964704, 0.44703123336599304 ], "peak_layer": 21, "peak_normalized": 3.152621923151655, "population_cv": 0.7020159090531873, "rest_mean": 0.052809414074376775, "spike_contrast": 3.2408441611351573, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.17114708125591277, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04836787283420563, 0.05606655776500702, 0.04991130903363228, 0.04613976180553436, 0.059560079127550125, 0.05521026998758316, 0.05200418829917908, 0.046326033771038055, 0.04069291427731514, 0.03640173375606537, 0.03422209620475769, 0.03272663801908493, 0.04618780314922333, 0.04299197345972061, 0.040533751249313354, 0.037876538932323456, 0.046885788440704346, 0.0442398302257061, 0.0448639914393425, 0.04783805459737778, 0.22478091716766357, 0.19202154874801636, 0.1533295214176178, 0.12133045494556427, 0.1642729640007019, 0.1439400315284729, 0.10835958272218704, 0.06995898485183716, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.317203296797743e-06, "first_to_last_ratio": 0.35664923571220175, "imbalance_abs_log_ratio": 1.0310025132235103, "last_quartile_mean": 6.49714920086808e-06, "mean": 4.492464356076198e-06, "normalized": [ 0.5694178936246215, 0.527096091999763, 0.5069565208451643, 0.5168980803173514, 0.5167362729922896, 0.5011287746943238, 0.4906181312174465, 0.49753075069099545, 0.4121289001968579, 0.3863295396034041, 0.3787441822616284, 0.3878491467448858, 0.49538413383836966, 0.4937248622259753, 0.4965317157240282, 0.4922453146709141, 0.5531637227610273, 0.5731353625097597, 0.5949767701689571, 0.6269395119699129, 2.991445440442326, 2.8607213177085975, 2.466076440453951, 2.094358907038209, 2.7670265254785567, 2.47746135973099, 1.9032918373610526, 1.2284996183916332, 1.2122250507961028, 0.87482980171944, 0.5899517341136686, 0.5165762877077978 ], "peak_layer": 21, "peak_normalized": 2.991445440442326, "population_cv": 0.8381751118691627, "rest_mean": 3.131475834918631e-06, "spike_contrast": 3.7815403964740346, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.184180237032706e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.5580895908206003e-06, 2.367960405535996e-06, 2.2774840999773005e-06, 2.322146201549913e-06, 2.321419287909521e-06, 2.2513031581183895e-06, 2.2040844669390935e-06, 2.23513916353113e-06, 1.8514743942432688e-06, 1.7355716863676207e-06, 1.7014947388815926e-06, 1.742398467285966e-06, 2.2254955638345564e-06, 2.2180413452588255e-06, 2.2306510345515562e-06, 2.2113945306045935e-06, 2.485068307578331e-06, 2.5747901872819057e-06, 2.67291193267738e-06, 2.8165034109406406e-06, 1.343896201433381e-05, 1.2851688552473206e-05, 1.107876050809864e-05, 9.408832738699857e-06, 1.2430768038029782e-05, 1.1129906852147542e-05, 8.550470738555305e-06, 5.518990747077623e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.480986444832524e-05, "first_to_last_ratio": 0.7542988096897616, "imbalance_abs_log_ratio": 0.28196669015672604, "last_quartile_mean": 5.940598589404544e-05, "mean": 5.624602624720865e-05, "normalized": [ 0.7593253658919704, 0.8584935198542845, 0.7758342247870141, 0.7257052209284447, 0.9115911301096014, 0.8478599752338415, 0.7912991525483561, 0.7033001562001767, 0.6342038809025199, 0.5708934677878957, 0.5360030772522035, 0.5119164360939763, 0.7122797103032084, 0.65763851786042, 0.6170495828015774, 0.5742094044317299, 0.7380398429228872, 0.6958734981683372, 0.7020379955965178, 0.7406705623157649, 3.198556354491257, 2.6580706472996987, 2.0667008648707084, 1.5629991771988188, 2.1587737798302467, 1.7912836314300635, 1.3377158632775958, 0.8975280712077295, 0.8407131217921131, 0.5976858891866252, 0.431587196610723, 0.394160680813692 ], "peak_layer": 21, "peak_normalized": 3.198556354491257, "population_cv": 0.6612253154131089, "rest_mean": 4.240304419740655e-05, "spike_contrast": 3.089356714727384, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013099812931613997, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.270903446013108e-05, 4.828684905078262e-05, 4.363759217085317e-05, 4.081803490407765e-05, 5.127337863086723e-05, 4.768875442096032e-05, 4.4507432903628796e-05, 3.955783904530108e-05, 3.567144813132472e-05, 3.211048897355795e-05, 3.0148043151712045e-05, 2.8793265300919302e-05, 4.006290328106843e-05, 3.698955333675258e-05, 3.4706587030086666e-05, 3.229699723306112e-05, 4.151180837652646e-05, 3.9140119042713195e-05, 3.948684752685949e-05, 4.16597758885473e-05, 0.00017990608466789126, 0.00014950591139495373, 0.00011624371109064668, 8.791249274509028e-05, 0.00012142244668211788, 0.00010075258614961058, 7.524120155721903e-05, 5.0482387450756505e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.384813680895604e-05, "first_to_last_ratio": 0.9740578591651334, "imbalance_abs_log_ratio": 0.026284573443628244, "last_quartile_mean": 3.474961624760908e-05, "mean": 3.869269224310301e-05, "normalized": [ 0.8541900771208318, 0.9413918806879924, 0.859570413488977, 0.8162196112220202, 0.9909206111274242, 0.9266644039643532, 0.8522558488291101, 0.7571395238293102, 0.7081640210424396, 0.6442698940442247, 0.603596002806512, 0.575557073332959, 0.7887193829441419, 0.7247641884135786, 0.6750858213648586, 0.6336813762759561, 0.8435498470464944, 0.7939958245882761, 0.7970984689300282, 0.8220042438012486, 3.299009695161972, 2.613224785716232, 1.9509379481481457, 1.3432491513722262, 1.9254306181237832, 1.4942554786145408, 1.1183073895840108, 0.8225722329666332, 0.6567052933181462, 0.46462952270788027, 0.36224409286636666, 0.34059527655932637 ], "peak_layer": 21, "peak_normalized": 3.299009695161972, "population_cv": 0.6253906355726564, "rest_mean": 2.9905363724672633e-05, "spike_contrast": 2.880562397960604, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.614426624262706e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.305091377114877e-05, 3.642498631961644e-05, 3.325909347040579e-05, 3.158173421979882e-05, 3.834138624370098e-05, 3.5855140595231205e-05, 3.297607327112928e-05, 2.9295766580617055e-05, 2.740077252383344e-05, 2.4928536731749773e-05, 2.3354754375759512e-05, 2.2269852706813253e-05, 3.051767635042779e-05, 2.804307769110892e-05, 2.612088792375289e-05, 2.4518838472431526e-05, 3.263921462348662e-05, 3.072183608310297e-05, 3.084188574575819e-05, 3.180555722792633e-05, 0.00012764756684191525, 0.00010111270239576697, 7.548704161308706e-05, 5.1973926019854844e-05, 7.450009434251115e-05, 5.781676736660302e-05, 4.327032365836203e-05, 3.182753425789997e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001188372980323038, "first_to_last_ratio": 0.585909807801185, "imbalance_abs_log_ratio": 0.534589412852804, "last_quartile_mean": 0.00020282524110371014, "mean": 0.00016756316449573205, "normalized": [ 0.6676704870164084, 0.7483084683488639, 0.6769676096236417, 0.6368287558643128, 0.8100066941936953, 0.7564679298832396, 0.7242231413041909, 0.6531986131482728, 0.5690738743162227, 0.5079946750459964, 0.47515152884923867, 0.45873382032473325, 0.6302192489862594, 0.5851397763517093, 0.5571686419883745, 0.5132471085839729, 0.6453685522588855, 0.614712668030142, 0.6206621143750464, 0.6587669430337698, 3.1211444947467313, 2.6936917077524942, 2.215351775191127, 1.776377347392615, 2.3967757154563434, 2.05632794722869, 1.5725614581941492, 1.0240120317645556, 1.032250695352248, 0.7057034519699571, 0.4689146276836353, 0.42697809574047824 ], "peak_layer": 21, "peak_normalized": 3.1211444947467313, "population_cv": 0.718404168086637, "rest_mean": 0.00012285891932185255, "spike_contrast": 3.3287456107546904, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004089660884346813, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011187697964487597, 0.00012538893497548997, 0.00011343483492964879, 0.00010670904157450423, 0.0001357272849418223, 0.00012675616017077118, 0.00012135312135796994, 0.00010945202666334808, 9.535581921227276e-05, 8.512119529768825e-05, 7.961789378896356e-05, 7.686689059482887e-05, 0.0001056015316862613, 9.804787259781733e-05, 9.336094080936164e-05, 8.60013096826151e-05, 0.00010813999688252807, 0.00010300319991074502, 0.00010400010796729475, 0.00011038507363991812, 0.000522988848388195, 0.0004513635067269206, 0.0003712113539222628, 0.0002976554096676409, 0.0004016113234683871, 0.000344564818078652, 0.0002635033742990345, 0.00017158669652417302, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4509995708067436e-05, "first_to_last_ratio": 0.7743031141222962, "imbalance_abs_log_ratio": 0.2557918617425859, "last_quartile_mean": 1.8739425740932347e-05, "mean": 1.7722179109114222e-05, "normalized": [ 0.7655666384169513, 0.8917483357831588, 0.7902102983812471, 0.7316333655654478, 0.9592978291511814, 0.8828085684394261, 0.8112146819967133, 0.7175029311175319, 0.593548092754179, 0.5514020456598471, 0.522638501525885, 0.5037614169375741, 0.7045039393419045, 0.6588959250307225, 0.6114268576480177, 0.579302599818883, 0.7435448955446501, 0.6887296875838754, 0.6921332531242588, 0.8093892960298869, 3.2325455196933226, 2.5123032009763855, 1.9971252826810495, 1.5895697479726458, 2.322878133149869, 1.7452609194969326, 1.1261946298991334, 0.8913309023592738, 0.8429252014113704, 0.5609036083922285, 0.5024807882604625, 0.46722290585598436 ], "peak_layer": 21, "peak_normalized": 3.2325455196933226, "population_cv": 0.6537085087930902, "rest_mean": 1.335436960895701e-05, "spike_contrast": 3.09324600258607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.130835040996317e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3567509085987695e-05, 1.580372372700367e-05, 1.4004248441779055e-05, 1.2966137546754908e-05, 1.700084794720169e-05, 1.564529156894423e-05, 1.437649189028889e-05, 1.2715715456579346e-05, 1.05189656096627e-05, 9.772045814315788e-06, 9.2622931333608e-06, 8.927750059228856e-06, 1.2485344996093772e-05, 1.167707159765996e-05, 1.0835816283361055e-05, 1.0266504432365764e-05, 1.3177235814509913e-05, 1.2205790881125722e-05, 1.2266109479242004e-05, 1.4344142073241528e-05, 5.7287750678369775e-05, 4.452348730410449e-05, 3.539341196301393e-05, 2.8170639780000784e-05, 4.116646232432686e-05, 3.092982660746202e-05, 1.9958622942795046e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.7723796544451034e-05, "first_to_last_ratio": 0.7709532556520535, "imbalance_abs_log_ratio": 0.2601275354642249, "last_quartile_mean": 7.487327684430056e-05, "mean": 7.085550987540046e-05, "normalized": [ 0.767459364760998, 0.8911005669273344, 0.7949447373790853, 0.7325731165004719, 0.9435545502986998, 0.8683878924182112, 0.8054854811328411, 0.7138473259026528, 0.5978031833741329, 0.5556350739964981, 0.526183189154655, 0.5057617239608265, 0.7151088387962502, 0.660810541031278, 0.6122576977156078, 0.5785452674352782, 0.739958743085955, 0.6892518806239643, 0.6970577547891826, 0.8113719765230756, 3.2405178430845596, 2.524955075289844, 1.989505542525185, 1.5842933275736053, 2.315101352895563, 1.736925913599177, 1.1174905309838439, 0.8878674587909959, 0.8457916652531124, 0.5666462793963278, 0.5093376021450956, 0.4744685026556913 ], "peak_layer": 21, "peak_normalized": 3.2405178430845596, "population_cv": 0.6532804064480114, "rest_mean": 5.3392583881591066e-05, "spike_contrast": 3.093225654863168, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016515531024197116, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.437872459879145e-05, 6.313938501989469e-05, 5.63262146897614e-05, 5.190684169065207e-05, 6.685603875666857e-05, 6.153006688691676e-05, 5.7073084462899715e-05, 5.058001625002362e-05, 4.2357649363111705e-05, 3.936980647267774e-05, 3.728297815541737e-05, 3.58360048267059e-05, 5.066940138931386e-05, 4.682206781581044e-05, 4.33818313467782e-05, 4.099311991012655e-05, 5.2430154028115794e-05, 4.883729343418963e-05, 4.93903826281894e-05, 5.749017509515397e-05, 0.0002296085440320894, 0.00017890697927214205, 0.0001409674296155572, 0.00011225591151742265, 0.00016403768677264452, 0.00012307077122386545, 7.918036135379225e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05119003914296627, "first_to_last_ratio": 0.7709532409156976, "imbalance_abs_log_ratio": 0.260127554578686, "last_quartile_mean": 0.06639837077818811, "mean": 0.06283537502167746, "normalized": [ 0.7674593499219178, 0.8911005434486201, 0.7949446960721035, 0.7325731497404293, 0.9435544639718605, 0.8683878801940695, 0.8054854824220893, 0.7138473266473999, 0.5978031662245904, 0.5556351000440413, 0.5261832202468875, 0.5057617435862689, 0.715108825077915, 0.6608105011251547, 0.6122576924651686, 0.5785453059609885, 0.7399587078512944, 0.6892519015892086, 0.6970578010795431, 0.8113719810435487, 3.2405178171891893, 2.52495521313829, 1.9895055108588098, 1.5842933381515236, 2.315101350530748, 1.7369258738398763, 1.1174905410660712, 0.8878674720175045, 0.8457916556516879, 0.5666462655907423, 0.5093376096374259, 0.4744685136150315 ], "peak_layer": 21, "peak_normalized": 3.2405178171891893, "population_cv": 0.6532804095990112, "rest_mean": 0.04734907733897368, "spike_contrast": 3.0932256918071666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1464613825082779, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048223596066236496, 0.05599263682961464, 0.049950648099184036, 0.04603150859475136, 0.05928859859704971, 0.05456547811627388, 0.050612982362508774, 0.04485486447811127, 0.03756318613886833, 0.03491353988647461, 0.03306291997432709, 0.03177972882986069, 0.04493413120508194, 0.041522275656461716, 0.038471441715955734, 0.036353111267089844, 0.04649558290839195, 0.04330940172076225, 0.04379988834261894, 0.05098286271095276, 0.20361915230751038, 0.158656507730484, 0.12501132488250732, 0.09954966604709625, 0.1454702615737915, 0.10914038866758347, 0.07021793723106384, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.293479383297381e-06, "first_to_last_ratio": 0.445969741543177, "imbalance_abs_log_ratio": 0.8075041733371805, "last_quartile_mean": 5.142679356140434e-06, "mean": 3.873292421019414e-06, "normalized": [ 0.6548345314656568, 0.6098593868708077, 0.5890922218088536, 0.6012559418227537, 0.5982935563608919, 0.5743111722490878, 0.5597275993993273, 0.5496382066817245, 0.4494881925876058, 0.4317042818847807, 0.42858151997472205, 0.44091762118925537, 0.5606916778798313, 0.5514610505577893, 0.5418155695170769, 0.543856847054833, 0.6362195338062625, 0.6499443412604623, 0.6710047275371834, 0.7226720079645516, 3.0150009122712964, 2.639960940380615, 2.3465929682489137, 2.0112500416813845, 2.8998499180306, 2.237539331592885, 1.506545124394083, 1.1285810413622663, 1.0054446853908898, 0.6799751247000538, 0.607691428375801, 0.5561984956977537 ], "peak_layer": 21, "peak_normalized": 3.0150009122712964, "population_cv": 0.7767874334961873, "rest_mean": 2.7381803546530946e-06, "spike_contrast": 3.653118598414905, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.000289757939754e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.5363656277477276e-06, 2.362163741054246e-06, 2.2817264380137203e-06, 2.3288400825549616e-06, 2.3173658973973943e-06, 2.224475110779167e-06, 2.167988668588805e-06, 2.128909500243026e-06, 1.7409992096872884e-06, 1.6721169231459498e-06, 1.6600215531070717e-06, 1.707802880446252e-06, 2.1717228264606092e-06, 2.135969907612889e-06, 2.0986101390008116e-06, 2.106516603816999e-06, 2.4642642983963015e-06, 2.5174244910886046e-06, 2.5989975256379694e-06, 2.799120011331979e-06, 1.1677980182867032e-05, 1.0225340702163521e-05, 9.089040759135969e-06, 7.790159543219488e-06, 1.1231966709601693e-05, 8.666644134791568e-06, 5.835289812239353e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.442123326953151e-05, "first_to_last_ratio": 0.9390502622041328, "imbalance_abs_log_ratio": 0.06288627383246957, "last_quartile_mean": 4.730442560685333e-05, "mean": 5.047756872045284e-05, "normalized": [ 0.8442457511608136, 0.9554189601166778, 0.8651399119276457, 0.8073037852085534, 1.011773735449473, 0.9346086902233326, 0.859731545150837, 0.7619317954676181, 0.657244201472419, 0.6123980431859172, 0.5782989976765822, 0.5555348776283999, 0.7735150062029652, 0.7093353168527977, 0.6548347172091762, 0.6154299343077795, 0.8164721063408712, 0.7613444152108153, 0.7654949233837004, 0.8721362954625393, 3.2652345954629065, 2.4905637834711873, 1.8895133502771215, 1.4453947777525853, 2.0912180007544503, 1.5360217136102785, 1.0169235828420913, 0.8253134404517026, 0.6717824033622655, 0.48834596354787496, 0.4437661443704546, 0.4237292344581673 ], "peak_layer": 21, "peak_normalized": 3.2652345954629065, "population_cv": 0.6182834768826335, "rest_mean": 3.892021615886026e-05, "spike_contrast": 2.900479074737969, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011288727255305275, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.2615472921170294e-05, 4.8227226216113195e-05, 4.367015935713425e-05, 4.075073229614645e-05, 5.107187826070003e-05, 4.717677438748069e-05, 4.3397158151492476e-05, 3.846046456601471e-05, 3.317608934594318e-05, 3.091236430918798e-05, 2.9191127396188676e-05, 2.8042049962095916e-05, 3.904515688191168e-05, 3.580552220228128e-05, 3.305446443846449e-05, 3.106540680164471e-05, 4.1213526856154203e-05, 3.843081503873691e-05, 3.8640322600258514e-05, 4.402331978781149e-05, 0.00016482110368087888, 0.00012571760453283787, 9.537803998682648e-05, 7.296001422218978e-05, 0.00010555960034253076, 7.753464160487056e-05, 5.133183003636077e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.35970580636058e-05, "first_to_last_ratio": 1.183082469649485, "imbalance_abs_log_ratio": 0.16812329486526428, "last_quartile_mean": 2.8397900337040483e-05, "mean": 3.5426135951865945e-05, "normalized": [ 0.9325726638234952, 1.0264228324198814, 0.9395008785893599, 0.8896754656866502, 1.0823546164520876, 1.002292895319797, 0.9051263930494292, 0.8090082728462467, 0.7240290452324956, 0.6814307833477395, 0.6400266234002182, 0.6142285583444715, 0.8434981841324323, 0.7676858556869983, 0.7064344235164509, 0.6656552980925776, 0.9160791051424548, 0.8560305037244728, 0.8526061894702537, 0.9377555262910077, 3.3385841431029775, 2.450559245049383, 1.764118224393788, 1.2414543227163002, 1.7869665660194411, 1.2750853185739486, 0.921819584871679, 0.7867281257526472, 0.5090598376007598, 0.4052896271109379, 0.3729632040504296, 0.3549576861891871 ], "peak_layer": 21, "peak_normalized": 3.3385841431029775, "population_cv": 0.5922680449883523, "rest_mean": 2.8102526947030264e-05, "spike_contrast": 2.6678606062470647, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.497362457797863e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.303744597360492e-05, 3.6362194805406034e-05, 3.328288585180417e-05, 3.151776400045492e-05, 3.834364179056138e-05, 3.5507364373188466e-05, 3.206513065379113e-05, 2.8660037060035393e-05, 2.5649551389506087e-05, 2.414045957266353e-05, 2.2673670173389837e-05, 2.175974441342987e-05, 2.98818813462276e-05, 2.7196143491892144e-05, 2.5026241928571835e-05, 2.3581595087307505e-05, 3.24531429214403e-05, 3.032585300388746e-05, 3.0204542781575583e-05, 3.3221054763998836e-05, 0.00011827313574030995, 8.681384497322142e-05, 6.249589205253869e-05, 4.3979929614579305e-05, 6.330532050924376e-05, 4.5171345846029e-05, 3.265650593675673e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001175618963316083, "first_to_last_ratio": 0.7472280359888956, "imbalance_abs_log_ratio": 0.2913848713729993, "last_quartile_mean": 0.00015733068175904918, "mean": 0.0001475783419664367, "normalized": [ 0.7539335055517485, 0.8485908925894718, 0.7671380034067055, 0.7221187547562506, 0.9161421981299216, 0.8510423035463365, 0.8040139845348296, 0.7098739762546249, 0.5921519684486223, 0.5526890935329054, 0.5267158319052243, 0.5064890538358937, 0.699155203785276, 0.6460998215976334, 0.6066243745851786, 0.5643569853524535, 0.729020737260662, 0.6850322033333521, 0.6876316692176827, 0.7819456161539314, 3.243720072824733, 2.5794511063238676, 2.043978587180185, 1.6534243806222502, 2.3371192579790736, 1.7969709048957057, 1.167004731785933, 0.9411888715768515, 0.8017105792030909, 0.5375092481175581, 0.48493150700127435, 0.4622245747107726 ], "peak_layer": 21, "peak_normalized": 3.243720072824733, "population_cv": 0.6705093936976806, "rest_mean": 0.00011009511891407547, "spike_contrast": 3.178957885701889, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003499877464491874, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011126425670227036, 0.00012523363693617284, 0.00011321295460220426, 0.00010656908852979541, 0.00013520274660550058, 0.00012559541210066527, 0.00011865505075547844, 0.0001047620244207792, 8.738880569580942e-05, 8.156494004651904e-05, 7.773184916004539e-05, 7.474681478925049e-05, 0.0001031801657518372, 9.535034041618928e-05, 8.952461939770728e-05, 8.328686817549169e-05, 0.00010758767166407779, 0.00010109591676155105, 0.00010147954162675887, 0.00011539823753992096, 0.00047870283015072346, 0.00038067111745476723, 0.0003016469709109515, 0.00024400962865911424, 0.00034490818507038057, 0.0002651939867064357, 0.00017222462338395417, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4657116139460413e-05, "first_to_last_ratio": 0.6094207983868245, "imbalance_abs_log_ratio": 0.49524628372702406, "last_quartile_mean": 2.4050895831351227e-05, "mean": 2.012364140568934e-05, "normalized": [ 0.6757810223737426, 0.7862782366272546, 0.6956742911341667, 0.645696070130233, 0.8486116417698539, 0.7866567023571445, 0.7355711504661757, 0.6525555016113147, 0.5676629357160193, 0.5065196141176592, 0.4765477406007062, 0.4567702622869, 0.6387850255773179, 0.6012516562554105, 0.5676978717094758, 0.5320979135959467, 0.660140815357474, 0.6199672249932086, 0.6251969579127784, 0.6703296515344952, 3.1476456930462797, 2.6843935159061667, 2.147025871519746, 1.7098926204474763, 2.310003501268429, 2.02536057377974, 1.5260208260637402, 0.9852869427838804, 1.0403527573808369, 0.7424129105715245, 0.4909974573147262, 0.4408150437901761 ], "peak_layer": 21, "peak_normalized": 3.1476456930462797, "population_cv": 0.7023455047176551, "rest_mean": 1.4907175278090614e-05, "spike_contrast": 3.239551262652627, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.829255849472247e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3599174963019323e-05, 1.582278127898462e-05, 1.3999499969941098e-05, 1.2993756172363646e-05, 1.7077156371669844e-05, 1.5830397387617268e-05, 1.4802370060351677e-05, 1.3131792911735829e-05, 1.1423445357650053e-05, 1.0193019079451915e-05, 9.589875844540074e-06, 9.191880963044241e-06, 1.2854680790042039e-05, 1.2099372725060675e-05, 1.1424148397054523e-05, 1.0707747605920304e-05, 1.3284437045513187e-05, 1.2475998119043652e-05, 1.2581239388964605e-05, 1.3489473531080876e-05, 6.334209319902584e-05, 5.4019772505853325e-05, 4.3205978727201e-05, 3.440926593611948e-05, 4.648568210541271e-05, 4.07576299039647e-05, 3.0709095881320536e-05, 1.982756111829076e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.829710562466062e-05, "first_to_last_ratio": 0.6072508868460227, "imbalance_abs_log_ratio": 0.49881325067107773, "last_quartile_mean": 9.600168050383218e-05, "mean": 8.04003983603252e-05, "normalized": [ 0.6783620403407459, 0.7863497478511254, 0.7000374993610626, 0.6471153960919661, 0.8353499122055624, 0.7743388664853165, 0.7293878791763712, 0.6497369450201427, 0.570732789612889, 0.5105438596817938, 0.4799740390322282, 0.4590044982477276, 0.6477850704647679, 0.6029712759057101, 0.5684996518209374, 0.5312286531130485, 0.6575848951936205, 0.6204730346458087, 0.629222556567805, 0.6709457586481146, 3.1526166331987144, 2.693185986758889, 2.1505035942092414, 1.701690725446724, 2.3039587527644554, 2.018829142146718, 1.519765537386825, 0.9812068005325093, 1.0399315417453956, 0.7451740498188497, 0.49646309627653795, 0.44702977024839613 ], "peak_layer": 21, "peak_normalized": 3.1526166331987144, "population_cv": 0.7020174312406612, "rest_mean": 5.955002699112002e-05, "spike_contrast": 3.2408449417296112, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019299240375403315, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.454057827591896e-05, 6.322283297777176e-05, 5.628329381579533e-05, 5.202833563089371e-05, 6.71624657115899e-05, 6.22571533313021e-05, 5.8643076044972986e-05, 5.223910920904018e-05, 4.588714364217594e-05, 4.1047929698834196e-05, 3.859010394080542e-05, 3.6904144508298486e-05, 5.2082177717238665e-05, 4.8479130782652646e-05, 4.5707598474109545e-05, 4.27109953307081e-05, 5.287008752929978e-05, 4.988627915736288e-05, 5.058974420535378e-05, 5.3944306273479015e-05, 0.00025347163318656385, 0.00021653322619386017, 0.00017290134564973414, 0.00013681661221198738, 0.0001852392015280202, 0.00016231466725002974, 0.00012218975462019444, 7.88894176366739e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05169845558702946, "first_to_last_ratio": 0.6072508754048147, "imbalance_abs_log_ratio": 0.49881326951206806, "last_quartile_mean": 0.08513525081798434, "mean": 0.0712998784147203, "normalized": [ 0.6783619951609938, 0.786349724122647, 0.7000374743682577, 0.6471153772114631, 0.8353498624040391, 0.774338855687226, 0.729387848399732, 0.6497369828883814, 0.5707327933468814, 0.5105438598068986, 0.4799740650385461, 0.45900445901159087, 0.6477850423892585, 0.6029712911228355, 0.5684996530360322, 0.5312286608963362, 0.6575848715023773, 0.6204730273795, 0.6292225631787356, 0.6709457289261637, 3.152616671814094, 2.693186128732524, 2.150503721956032, 1.7016907445638214, 2.303958774916342, 2.0188290625883343, 1.5197655429953278, 0.981206773145755, 1.039931555380646, 0.7451740672617071, 0.49646307443703724, 0.4470297463304832 ], "peak_layer": 21, "peak_normalized": 3.152616671814094, "population_cv": 0.70201745433721, "rest_mean": 0.052809559360698415, "spike_contrast": 3.2408450927884243, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.17114760130643844, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048367127776145935, 0.056066639721393585, 0.04991258680820465, 0.046139247715473175, 0.059560343623161316, 0.05521026626229286, 0.05200526490807533, 0.0463261678814888, 0.04069317877292633, 0.036401715129613876, 0.03422209247946739, 0.0327269621193409, 0.04618699476122856, 0.04299177974462509, 0.04053395614027977, 0.037876538932323456, 0.04688572138547897, 0.044239651411771774, 0.044863492250442505, 0.047838348895311356, 0.22478118538856506, 0.1920238435268402, 0.15333065390586853, 0.12133034318685532, 0.1642719805240631, 0.14394226670265198, 0.10835909843444824, 0.06995992362499237, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.317259230721902e-06, "first_to_last_ratio": 0.35665874650259244, "imbalance_abs_log_ratio": 1.0309758465083512, "last_quartile_mean": 6.497132773120029e-06, "mean": 4.492468697492313e-06, "normalized": [ 0.5694283261976025, 0.527162238879597, 0.5069677223499952, 0.5169219758743683, 0.5167285360874581, 0.5011532928366966, 0.49061467097646116, 0.4975013703289752, 0.41211554520475835, 0.3863198030086292, 0.3787459166580058, 0.38785476948100367, 0.4953856795983567, 0.493696852070983, 0.4965364995551244, 0.49228294995504224, 0.553135351492583, 0.5731144625451857, 0.5949797886628256, 0.6269543934401883, 2.991422102255377, 2.8607693678126824, 2.46602465979906, 2.0943619443179213, 2.767059684919099, 2.477440137836224, 1.9032189385510152, 1.2285251544343567, 1.2121681046978803, 0.874835232215944, 0.589970244793492, 0.5166042831631082 ], "peak_layer": 21, "peak_normalized": 2.991422102255377, "population_cv": 0.8381704476164953, "rest_mean": 3.1314773423218867e-06, "spike_contrast": 3.7815448495732364, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1841822015412617e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.5581389309081715e-06, 2.3682598566665547e-06, 2.277536623296328e-06, 2.3222557956614764e-06, 2.3213867734739324e-06, 2.2514154807140585e-06, 2.2040710518922424e-06, 2.235009333162452e-06, 1.8514161865823553e-06, 1.7355296222376637e-06, 1.7015041748891235e-06, 1.742425411066506e-06, 2.225504658781574e-06, 2.2179176539793843e-06, 2.2306746814138023e-06, 2.211565742982202e-06, 2.484943252056837e-06, 2.5746987830643775e-06, 2.6729280762083363e-06, 2.8165729872853262e-06, 1.3438870155368932e-05, 1.2851916835643351e-05, 1.1078538591391407e-05, 9.408855476067401e-06, 1.2430929018591996e-05, 1.1129822269140277e-05, 8.550151505914982e-06, 5.519110800378257e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.480992902244907e-05, "first_to_last_ratio": 0.7542989526231301, "imbalance_abs_log_ratio": 0.28196650066503776, "last_quartile_mean": 5.940606024523731e-05, "mean": 5.624609735832564e-05, "normalized": [ 0.7593189081163986, 0.8584899766461837, 0.7758411348304551, 0.7257032038756371, 0.9115949579325926, 0.8478585798985628, 0.791306819195711, 0.703296291763442, 0.6342085768594776, 0.5708926166563143, 0.5360034021261918, 0.5119200900833328, 0.7122730532881045, 0.6576365221836818, 0.6170512605021473, 0.5742091312248548, 0.7380382630332617, 0.6958730711433307, 0.702031804287528, 0.7406728598801368, 3.1985536041966367, 2.658074789581204, 2.066704719937707, 1.5629982359965144, 2.1587687220561924, 1.7912910686863504, 1.3377069279004006, 0.8975311406544447, 0.8407126410078246, 0.5976870739310991, 0.4315898849446666, 0.3941606675796141 ], "peak_layer": 21, "peak_normalized": 3.1985536041966367, "population_cv": 0.6612253265050619, "rest_mean": 4.240309937341846e-05, "spike_contrast": 3.089356401125345, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013099828647682442, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.2708725231932476e-05, 4.828671080758795e-05, 4.363803600426763e-05, 4.081797305843793e-05, 5.1273658755235374e-05, 4.7688736231066287e-05, 4.450792039278895e-05, 3.9557671698275954e-05, 3.5671757359523326e-05, 3.2110481697600335e-05, 3.0148099540383555e-05, 2.8793507226509973e-05, 4.00625795009546e-05, 3.698948785313405e-05, 3.4706725273281336e-05, 3.229702269891277e-05, 4.151177199673839e-05, 3.9140144508564845e-05, 3.948654921259731e-05, 4.1659957787487656e-05, 0.0001799061574274674, 0.00014950633340049535, 0.0001162440748885274, 8.791255095275119e-05, 0.00012142231571488082, 0.00010075313184643164, 7.524079410359263e-05, 5.0482623919378966e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.384801061656617e-05, "first_to_last_ratio": 0.9740047819331958, "imbalance_abs_log_ratio": 0.02633906576931732, "last_quartile_mean": 3.475138032627001e-05, "mean": 3.8690275459885015e-05, "normalized": [ 0.8541538250770527, 0.9415230863648691, 0.8597053468534278, 0.816066837065049, 0.9910309334070734, 0.9267294340872885, 0.8523049476412223, 0.7572490181408758, 0.7081191176405685, 0.6443517927963444, 0.6037734323315225, 0.5756116900407472, 0.7887443909138365, 0.7248706260931408, 0.6750345264379841, 0.633794207108887, 0.8434851920289833, 0.7938278400728305, 0.7971348135467743, 0.8220444008320537, 3.298671908004925, 2.612487605975051, 1.950851070533051, 1.3428804993059804, 1.9255864325842058, 1.4942946568254274, 1.1186305566147283, 0.822664893239064, 0.6568214429021311, 0.4646495190211241, 0.3623372415841501, 0.34056871492963064 ], "peak_layer": 21, "peak_normalized": 3.298671908004925, "population_cv": 0.6252780463659825, "rest_mean": 2.9905465689134196e-05, "spike_contrast": 2.8800169546676924, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.612824822193944e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.304744677734561e-05, 3.64277875632979e-05, 3.3262236684095114e-05, 3.157385071972385e-05, 3.834325980278663e-05, 3.5855417081620544e-05, 3.297591320006177e-05, 2.9298173103597946e-05, 2.7397323719924316e-05, 2.4930148356361315e-05, 2.336016041226685e-05, 2.227057484560646e-05, 3.0516737751895562e-05, 2.8045444196322933e-05, 2.611727177281864e-05, 2.452167245792225e-05, 3.263467442593537e-05, 3.071341780014336e-05, 3.084136551478878e-05, 3.1805124308448285e-05, 0.00012762652477249503, 0.00010107786511071026, 7.547896530013531e-05, 5.195641642785631e-05, 7.450146949850023e-05, 5.781467189081013e-05, 4.3280124373268336e-05, 3.182913133059628e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011881819409609307, "first_to_last_ratio": 0.5858660432005869, "imbalance_abs_log_ratio": 0.5346641107578359, "last_quartile_mean": 0.0002028077842624043, "mean": 0.0001675532937497337, "normalized": [ 0.6676405577533395, 0.7480364199301687, 0.6769763115579581, 0.6368334430352999, 0.8101874659576359, 0.7560101568775733, 0.7242722329347805, 0.6531372168922783, 0.569150085590359, 0.5080318969181871, 0.47519098467081156, 0.45888126162494614, 0.6302390928396696, 0.5851986956962459, 0.5573344319393583, 0.5133335361358992, 0.6455392341889507, 0.6148024241191441, 0.6209170612517536, 0.6589570869141987, 3.121371094689797, 2.6936365726462816, 2.2146820529210776, 1.7763796870270414, 2.3963701078287363, 2.056340004797266, 1.572482658633456, 1.024072444271845, 1.0322195327669619, 0.7057443743535575, 0.46898398312387723, 0.42704789011154415 ], "peak_layer": 21, "peak_normalized": 3.121371094689797, "population_cv": 0.7183385489551041, "rest_mean": 0.00012285727657231123, "spike_contrast": 3.328348128139876, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004089117865078151, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011186537449248135, 0.00012533596600405872, 0.0001134296107920818, 0.0001067035409505479, 0.00013574957847595215, 0.0001266719918930903, 0.00012135419819969684, 0.00010943529196083546, 9.53629714786075e-05, 8.512241765856743e-05, 7.96198146417737e-05, 7.6887066825293e-05, 0.00010559863585513085, 9.805196896195412e-05, 9.338321979157627e-05, 8.601072477176785e-05, 0.0001081622249330394, 0.00010301217116648331, 0.00010403669875813648, 0.00011041043035220355, 0.0005229960079304874, 0.00045132767991162837, 0.0003710772725753486, 0.0002976382675115019, 0.00040151970461010933, 0.0003445465408731252, 0.0002634746488183737, 0.00017158671107608825, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4510050277749542e-05, "first_to_last_ratio": 0.7743096996079971, "imbalance_abs_log_ratio": 0.255783356730045, "last_quartile_mean": 1.873933683782525e-05, "mean": 1.772222358908948e-05, "normalized": [ 0.7655738518273417, 0.8917418894414295, 0.7902270979958571, 0.7316384060873433, 0.9592931634098782, 0.8828162060633833, 0.8112185990320713, 0.7174816289091832, 0.5935539930427093, 0.5514073845754808, 0.522640268952773, 0.50377149417381, 0.7044978089956251, 0.6589104369318574, 0.6114200884798895, 0.5793087411391302, 0.7435486231843338, 0.6887380175928803, 0.692133312159999, 0.8093969639662545, 3.2325450017939272, 2.512318244382783, 1.9971229388240013, 1.5895601132651613, 2.322861628648705, 1.745234574445335, 1.1261863634676146, 0.8913245597041289, 0.8429168761495727, 0.5609154923495704, 0.5024744978075042, 0.4672217332004658 ], "peak_layer": 21, "peak_normalized": 3.2325450017939272, "population_cv": 0.6537061924560114, "rest_mean": 1.3354412287838016e-05, "spike_contrast": 3.0932401760178796, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1308404615847397e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.356767097604461e-05, 1.5803649148438126e-05, 1.4004581316839904e-05, 1.2966259419044945e-05, 1.7000807929434814e-05, 1.564546619192697e-05, 1.4376597391674295e-05, 1.2715369848592672e-05, 1.0519096576899756e-05, 9.77216495812172e-06, 9.262347703042906e-06, 8.927951057557948e-06, 1.2485267689044122e-05, 1.1677358088491019e-05, 1.0835723514901474e-05, 1.0266639037581626e-05, 1.3177334949432407e-05, 1.220596914208727e-05, 1.2266141311556567e-05, 1.4344313967740163e-05, 5.728788528358564e-05, 4.452386565390043e-05, 3.5393459256738424e-05, 2.817053973558359e-05, 4.116627314942889e-05, 3.092943734372966e-05, 1.9958526536356658e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.772373378931661e-05, "first_to_last_ratio": 0.7709541968266947, "imbalance_abs_log_ratio": 0.26012631467157654, "last_quartile_mean": 7.487310404030723e-05, "mean": 7.085559036568156e-05, "normalized": [ 0.767456541889667, 0.8910969874796977, 0.7949526654342541, 0.7325741840266811, 0.943544133913626, 0.868390294625501, 0.8054913948148298, 0.7138323441642012, 0.5978064063949543, 0.5556378314836203, 0.5261854153191788, 0.505768953649729, 0.715108334510866, 0.6608161569684796, 0.612256437426044, 0.5785464585899268, 0.739960161627241, 0.6892568994732284, 0.697060043562065, 0.8113764458982461, 3.2405349047340315, 2.5249733605503017, 1.9894958890200054, 1.5842775624032772, 2.3150907133984013, 1.7369124395330813, 1.1174777605817472, 0.8878712764898554, 0.8457898829566253, 0.5666580094993431, 0.5093321459106411, 0.474467963670652 ], "peak_layer": 21, "peak_normalized": 3.2405349047340315, "population_cv": 0.6532808754860918, "rest_mean": 5.3392646400930564e-05, "spike_contrast": 3.093225357948515, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016515548777533694, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.437858635559678e-05, 6.313920312095433e-05, 5.632684042211622e-05, 5.1906976295867935e-05, 6.685537664452568e-05, 6.153030699351802e-05, 5.707356831408106e-05, 5.057901216787286e-05, 4.2357925849501044e-05, 3.9370046579279006e-05, 3.728317824425176e-05, 3.583655779948458e-05, 5.0669423217186704e-05, 4.682251892518252e-05, 4.338179132901132e-05, 4.099325087736361e-05, 5.2430314099183306e-05, 4.8837704525794834e-05, 4.9390600906917825e-05, 5.749055708292872e-05, 0.00022961001377552748, 0.00017890847811941057, 0.00014096690574660897, 0.00011225492198718712, 0.0001640371192479506, 0.00012306995631661266, 7.917954644653946e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051189984660595655, "first_to_last_ratio": 0.7709541992191667, "imbalance_abs_log_ratio": 0.26012631156831567, "last_quartile_mean": 0.06639821757562459, "mean": 0.06283544522011653, "normalized": [ 0.7674565953661515, 0.8910969393261454, 0.7949526416572095, 0.7325742284914066, 0.9435441611668938, 0.8683903486633082, 0.8054914004920491, 0.7138323596913118, 0.5978064112769625, 0.5556377993412199, 0.5261854188690636, 0.5057690043708923, 0.7151083226055974, 0.6608161658178341, 0.6122564748857159, 0.5785464975021892, 0.7399601340699639, 0.6892568823568349, 0.6970600459500164, 0.8113764696625058, 3.2405348286343636, 2.524973261140758, 1.9894956995598303, 1.584277576610783, 2.3150909383371494, 1.736912431812052, 1.1174776724854902, 0.8878712823128355, 0.8457898807407633, 0.5666580234128831, 0.5093321198410424, 0.47446798354877795 ], "peak_layer": 21, "peak_normalized": 3.2405348286343636, "population_cv": 0.6532808615381831, "rest_mean": 0.04734913239048587, "spike_contrast": 3.0932253054239998, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14646153450012206, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048223476856946945, 0.05599247291684151, 0.04995120316743851, 0.04603162780404091, 0.05928801745176315, 0.05456569418311119, 0.0506134107708931, 0.044853974133729935, 0.03756343200802803, 0.03491374850273132, 0.033063095062971115, 0.031780220568180084, 0.04493414983153343, 0.04152267798781395, 0.03847140818834305, 0.036353226751089096, 0.046495724469423294, 0.043309763073921204, 0.043800078332424164, 0.05098320171236992, 0.20362044870853424, 0.15865781903266907, 0.12501084804534912, 0.09954878687858582, 0.1454697698354721, 0.10913966596126556, 0.07021720707416534, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.293416855536634e-06, "first_to_last_ratio": 0.44595751640176395, "imbalance_abs_log_ratio": 0.8075315862036556, "last_quartile_mean": 5.1426801235265884e-06, "mean": 3.873313932700739e-06, "normalized": [ 0.6547872198845468, 0.6098622810029859, 0.5891095547173398, 0.6012100431571663, 0.5982908205734621, 0.5743244780654446, 0.5596766481388143, 0.5495961168488454, 0.44946743969713066, 0.4316777868522346, 0.4285841881361952, 0.44095385743945303, 0.560667078741325, 0.5514542308757108, 0.5417913687261439, 0.5438534743570708, 0.636286678312979, 0.6499450755816364, 0.6710355180394164, 0.7226628285450687, 3.0150865448734856, 2.639875835373709, 2.3467426593605225, 2.0112905298698336, 2.8997859114535975, 2.237494500860103, 1.506591468158109, 1.1286195048247083, 1.0054186728222039, 0.6799993493915261, 0.607662928646794, 0.5561954066724383 ], "peak_layer": 21, "peak_normalized": 3.0150865448734856, "population_cv": 0.7767952241449113, "rest_mean": 2.738177386163443e-06, "spike_contrast": 3.6531786927134653, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.000305128400214e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.536196461733198e-06, 2.3621880700375186e-06, 2.281806246173801e-06, 2.328675236640265e-06, 2.3173681711341487e-06, 2.2245390027819667e-06, 2.1678033590433188e-06, 2.128758296748856e-06, 1.7409284964742255e-06, 1.6720235862521804e-06, 1.6600411072431598e-06, 1.707952719698369e-06, 2.171639607695397e-06, 2.1359553556976607e-06, 2.0985280571039766e-06, 2.1065152395749465e-06, 2.464538056301535e-06, 2.5174413167405874e-06, 2.59913122135913e-06, 2.79910000244854e-06, 1.1678376722557005e-05, 1.022506785375299e-05, 9.089671038964298e-06, 7.790359632053878e-06, 1.1231781172682531e-05, 8.666518624522723e-06, 5.835501724504866e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.4421208258427214e-05, "first_to_last_ratio": 0.9390471381412793, "imbalance_abs_log_ratio": 0.06288960067042051, "last_quartile_mean": 4.730455634671671e-05, "mean": 5.047763789889359e-05, "normalized": [ 0.8442417833686091, 0.9554185876612655, 0.8651397352686891, 0.8073036157415787, 1.0117688173530386, 0.9346106525630644, 0.8597336821790709, 0.7619236882899906, 0.6572438052320593, 0.6123997263969396, 0.5783007276195616, 0.5555384405465414, 0.7735135857629635, 0.7093370113557148, 0.654834684626451, 0.6154293791596152, 0.8164712035979397, 0.7613467591474443, 0.7654961084945793, 0.8721378389215614, 3.265225507983762, 2.4905834329623366, 1.889506148192275, 1.4453841483405094, 2.0912250805998966, 1.5360203292388628, 1.0169178649058275, 0.8253190119907806, 0.6717866718190295, 0.48834958251132726, 0.44376373442170963, 0.42372865374700563 ], "peak_layer": 21, "peak_normalized": 3.265225507983762, "population_cv": 0.6182834965857263, "rest_mean": 3.8920269852732655e-05, "spike_contrast": 2.9004789991258666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011288742534816265, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.261533103999682e-05, 4.822727350983769e-05, 4.367021028883755e-05, 4.0750779589870945e-05, 5.1071699999738485e-05, 4.717693809652701e-05, 4.33973254985176e-05, 3.846010804409161e-05, 3.317611481179483e-05, 3.091249163844623e-05, 2.9191254725446925e-05, 2.8042268240824342e-05, 3.9045138692017645e-05, 3.5805656807497144e-05, 3.305450809421018e-05, 3.106542135355994e-05, 4.1213537770090625e-05, 3.843098602374084e-05, 3.8640435377601534e-05, 4.402345803100616e-05, 0.00016482087085023522, 0.00012571876868605614, 9.537780715618283e-05, 7.295957766473293e-05, 0.00010556010238360614, 7.753467798465863e-05, 5.1331611757632345e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.359962329341215e-05, "first_to_last_ratio": 1.183161693959634, "imbalance_abs_log_ratio": 0.1681902569407737, "last_quartile_mean": 2.839816693267494e-05, "mean": 3.542719008464701e-05, "normalized": [ 0.9327605618775979, 1.0266348424598963, 0.9395266301163642, 0.8894102418069995, 1.082225986222851, 1.0025232858328599, 0.9050487327854893, 0.8091772559917478, 0.7238157816582237, 0.6814271430375796, 0.6400264742225962, 0.6141081065852076, 0.843581628077757, 0.7675711067089205, 0.7064606405143428, 0.6655561644133133, 0.9162025945827288, 0.8560421033047051, 0.8525829253737746, 0.9377293691727334, 3.3385973508618734, 2.4504448425731913, 1.7642485194416198, 1.2415583752024149, 1.7870101279275883, 1.2751219306613844, 0.9216663623251447, 0.7867121103297002, 0.5090691818722745, 0.4053411321779632, 0.37287136742526056, 0.3549471244558967 ], "peak_layer": 21, "peak_normalized": 3.3385973508618734, "population_cv": 0.592278758078665, "rest_mean": 2.8103131289417097e-05, "spike_contrast": 2.6679271717711734, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.497710757888853e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.304508572909981e-05, 3.6370787711348385e-05, 3.3284788514720276e-05, 3.150930569972843e-05, 3.834022572846152e-05, 3.5516583011485636e-05, 3.2063333492260426e-05, 2.8666876460192725e-05, 2.5642759283073246e-05, 2.414104892523028e-05, 2.2674339561490342e-05, 2.1756124624516815e-05, 2.9885726689826697e-05, 2.7192887500859797e-05, 2.5027915398823097e-05, 2.3578784748679027e-05, 3.2458483474329114e-05, 3.032716631423682e-05, 3.0204617360141128e-05, 3.322111660963856e-05, 0.00011827712296508253, 8.681237522978336e-05, 6.250236765481532e-05, 4.398492455948144e-05, 6.330874748528004e-05, 4.517398701864295e-05, 3.265204941271804e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011756234653148567, "first_to_last_ratio": 0.7472422235887175, "imbalance_abs_log_ratio": 0.29136588457853974, "last_quartile_mean": 0.00015732829706394114, "mean": 0.0001475833446420438, "normalized": [ 0.7536685945005097, 0.8489437148344299, 0.7668873856587688, 0.7219862591209203, 0.9160304874966925, 0.8511301995015967, 0.8040925298742102, 0.7099228291391, 0.592249280993652, 0.5528729353184129, 0.5267845496399454, 0.5066519805725661, 0.6991026141257565, 0.6459439706463571, 0.6067103504076782, 0.5642338800686106, 0.7290006596953322, 0.6851008790441766, 0.687775193830433, 0.7817329016891325, 3.243635755848753, 2.5800278485304857, 2.0438475774053395, 1.653426311601236, 2.3369623381128184, 1.7968342666222143, 1.1670758041878126, 0.9412043950671267, 0.8016088114740478, 0.5374717514485083, 0.4848750369936043, 0.46220890654977165 ], "peak_layer": 21, "peak_normalized": 3.243635755848753, "population_cv": 0.6705219269582756, "rest_mean": 0.0001100977226289817, "spike_contrast": 3.179045807260368, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003500057035125792, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.0001112289319280535, 0.00012528995284810662, 0.00011317980533931404, 0.00010655314690666273, 0.00013519084313884377, 0.00012561264156829566, 0.00011867066496051848, 0.00010477278556209058, 8.74061297508888e-05, 8.159483695635572e-05, 7.774462574161589e-05, 7.47733938624151e-05, 0.00010317590204067528, 9.533057163935155e-05, 8.954034274211153e-05, 8.327152318088338e-05, 0.00010758835560409352, 0.00010110947914654389, 0.00010150416346732527, 0.00011537075624801219, 0.00047870661364868283, 0.0003807691391557455, 0.00030163786141201854, 0.00024401818518526852, 0.0003448967181611806, 0.0002651828108355403, 0.0001722409506328404, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.220272659149487e-06, "first_to_last_ratio": 1.069429605014154, "imbalance_abs_log_ratio": 0.06712542691043744, "last_quartile_mean": 8.621673288189413e-06, "mean": 1.2944014976312701e-05, "normalized": [ 0.6303729714265094, 0.8682643012395536, 0.6280898219637274, 0.5502831812812704, 0.33676078665107495, 0.7338236582315528, 0.9360409006501473, 1.0149193793174633, 0.6122272943195698, 0.5131421981111545, 0.43080038533307174, 0.37792854873844645, 0.39300605492101337, 0.6863011395079349, 0.6243318369244146, 0.5414566873457767, 0.793386920098201, 1.135440497453951, 1.3017177042809938, 1.597588130028887, 2.6171146314958285, 4.210609056409907, 2.860451118900051, 2.2773498949645425, 1.2385768946194602, 1.256602631548737, 0.8025963162001695, 0.5822644883727114, 0.47699344412607797, 0.2914573895713198, 0.32576029794875644, 0.3543414380177253 ], "peak_layer": 22, "peak_normalized": 4.210609056409907, "population_cv": 0.8557638034672043, "rest_mean": 9.010903349243799e-06, "spike_contrast": 3.7934951067201736, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4182817762484775e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.159557182807475e-06, 1.1238826118642464e-05, 8.130004061968066e-06, 7.122873739717761e-06, 4.359036665846361e-06, 9.498624422121793e-06, 1.2116127436456736e-05, 1.3137131645635236e-05, 7.924679266579915e-06, 6.642120297328802e-06, 5.5762866395525634e-06, 4.891912794846576e-06, 5.087076260679169e-06, 8.883492228051182e-06, 8.081360647338443e-06, 7.008623470028397e-06, 1.0269612175761722e-05, 1.4697158803755883e-05, 1.684945345914457e-05, 2.0679204681073315e-05, 3.38759709848091e-05, 5.450218668556772e-05, 3.7025722122052684e-05, 2.9478051146725193e-05, 1.603215787326917e-05, 1.6265483282040805e-05, 1.0388818736828398e-05, 7.536840257671429e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.628570084401872e-05, "first_to_last_ratio": 1.065353016422061, "imbalance_abs_log_ratio": 0.0633062150559101, "last_quartile_mean": 3.405979077797383e-05, "mean": 5.163632124549622e-05, "normalized": [ 0.5687040132298282, 0.8764593520729407, 0.6051388108071001, 0.5571670858837222, 0.3412536542304683, 0.7524665950392966, 0.8942377527803768, 1.0263056508948738, 0.5946205407803693, 0.5239558700278845, 0.42939602856142756, 0.3883393864768877, 0.40120729316033693, 0.7381389545660257, 0.6270734263766993, 0.5557805536677036, 0.7656375942833815, 1.1752771944325544, 1.2670292024807646, 1.589140645161745, 2.652708081192408, 4.256645166896803, 2.872494573593182, 2.263949247584013, 1.2185560925718533, 1.2799157794404965, 0.7966972545457802, 0.5211011847842488, 0.47241308488259476, 0.29979401599371164, 0.32815719518376885, 0.36023871841675353 ], "peak_layer": 22, "peak_normalized": 4.256645166896803, "population_cv": 0.8639609553478637, "rest_mean": 3.5831106625092046e-05, "spike_contrast": 3.823060270757878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013698448019567878, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.9365783120738342e-05, 4.5257136662257835e-05, 3.124714203295298e-05, 2.8770058634108864e-05, 1.762108331604395e-05, 3.885460682795383e-05, 4.6175147872418165e-05, 5.2994648285675794e-05, 3.0704017262905836e-05, 2.7055153623223305e-05, 2.2172431272338144e-05, 2.0052417312399484e-05, 2.0716868675663136e-05, 3.8114780181786045e-05, 3.2379764888901263e-05, 2.86984632111853e-05, 3.953470877604559e-05, 6.06869907642249e-05, 6.542472692672163e-05, 8.205737685784698e-05, 0.00013697608665097505, 0.0002197974972659722, 0.00014832505257800221, 0.00011690201063174754, 6.292175385169685e-05, 6.609014235436916e-05, 4.113851537113078e-05, 2.690774817892816e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.032178521854802966, "first_to_last_ratio": 1.0653530079886349, "imbalance_abs_log_ratio": 0.06330620713982399, "last_quartile_mean": 0.030204562819562852, "mean": 0.04579160569119267, "normalized": [ 0.5687040352411169, 0.8764592932397164, 0.6051388058773625, 0.5571671042879097, 0.3412536282040628, 0.7524665522800605, 0.8942377987777075, 1.0263056510911557, 0.5946205787763836, 0.523955869943047, 0.42939603938950854, 0.388339353567458, 0.4012073030262001, 0.7381389650225099, 0.627073439274379, 0.5557805622734189, 0.7656375421356683, 1.1752771374183286, 1.2670292275109216, 1.589140749744225, 2.6527080853277174, 4.256645179045396, 2.872494621066323, 2.2639491530094316, 1.2185561059252055, 1.2799157254825457, 0.796697271015912, 0.52110118872801, 0.47241309150505434, 0.299794020683871, 0.32815719848905306, 0.3602387226403389 ], "peak_layer": 22, "peak_normalized": 4.256645179045396, "population_cv": 0.8639609572615631, "rest_mean": 0.031775383451194676, "spike_contrast": 3.823060262160724, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12147920578718185, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.026041870936751366, 0.040134478360414505, 0.02771027758717537, 0.025513576343655586, 0.01562655158340931, 0.03445665165781975, 0.04094858467578888, 0.046996183693408966, 0.02722863107919693, 0.023992780596017838, 0.019662734121084213, 0.017782682552933693, 0.018371926620602608, 0.03380056843161583, 0.028714699670672417, 0.02545008435845375, 0.03505977243185043, 0.05381782725453377, 0.058019302785396576, 0.07276930660009384, 0.12147176265716553, 0.19491861760616302, 0.13153614103794098, 0.10366986691951752, 0.05579964071512222, 0.05860939621925354, 0.03648204728960991, 0.02386206015944481, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.4475698694459425e-06, "first_to_last_ratio": 0.4032796404419227, "imbalance_abs_log_ratio": 0.9081250607923619, "last_quartile_mean": 6.069162992616839e-06, "mean": 4.194480464292383e-06, "normalized": [ 1.9615731362727682, 0.7747288452238285, 0.1785692992504359, 0.2633308680375469, 0.18148950204723951, 0.349408887478932, 0.383871487008665, 0.575200617814069, 0.3462103003815746, 0.3364194451166286, 0.2854708899046967, 0.31108734109147806, 0.30067908576983693, 0.513261777831553, 0.5156365685672093, 0.5885731468966555, 0.3799425036372034, 0.7326093628890369, 0.5694277551741158, 0.8619428981160375, 1.5998405056629517, 3.0497744670188736, 2.8846131894263896, 2.4808153927254097, 1.8266821603141778, 2.5325708585606295, 2.0509024328203003, 1.5887038326529273, 1.3182729046192574, 0.967953515357782, 0.6305408473131235, 0.6598961750186662 ], "peak_layer": 22, "peak_normalized": 3.0497744670188736, "population_cv": 0.8409982900394547, "rest_mean": 3.131610654876298e-06, "spike_contrast": 3.1721623566680712, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.93397743513924e-06, "top_five_layers": [ 22, 23, 26, 24, 27 ], "top_five_spike_overlap": 3 }, "values": [ 8.227780199376866e-06, 3.2495850064151455e-06, 7.490054372283339e-07, 1.1045361816286459e-06, 7.612541708112985e-07, 1.4655887525805156e-06, 1.6101414530567126e-06, 2.4126677544700215e-06, 1.4521723414873122e-06, 1.411104790349782e-06, 1.197402070829412e-06, 1.3048497748968657e-06, 1.2611925512828748e-06, 2.1528665001824265e-06, 2.162827513529919e-06, 2.4687585664651124e-06, 1.5936614090605872e-06, 3.072915660595754e-06, 2.3884535949036945e-06, 3.615402647483279e-06, 6.710499746986898e-06, 1.279221942240838e-05, 1.2099453670089133e-05, 1.0405731700302567e-05, 7.661982635909226e-06, 1.0622818990668748e-05, 8.60247018863447e-06, 6.663787189609138e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4055149197010905e-05, "first_to_last_ratio": 1.0862063963884478, "imbalance_abs_log_ratio": 0.08269125537722917, "last_quartile_mean": 2.2146020569380198e-05, "mean": 3.472467025744663e-05, "normalized": [ 0.4666666997881285, 0.8663177887052829, 0.6219999695375312, 0.5679169854003736, 0.3844149632749681, 0.7928386669686874, 0.9321622159542684, 0.9095983613860097, 0.6255219017157577, 0.5547521439337422, 0.4549635591797121, 0.4053293755903786, 0.4456534403712468, 0.7958744306526087, 0.6527431991239231, 0.5563097060509357, 0.7807692106312896, 1.1853454205486673, 1.2646459398653336, 1.548550635017449, 2.506691252154866, 4.290967865992318, 3.0302171410000085, 2.2576657022736453, 1.197190310100688, 1.343054897090299, 0.7711430639674054, 0.472191241764576, 0.47986836664550414, 0.29867429600060663, 0.25046027339059757, 0.2895009759231917 ], "peak_layer": 22, "peak_normalized": 4.290967865992318, "population_cv": 0.8713516434982468, "rest_mean": 2.4072257407102302e-05, "spike_contrast": 3.832116701364666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.224769964930602e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.6204847270273603e-05, 3.0082599550951272e-05, 2.1598743842332624e-05, 1.9720730051631108e-05, 1.3348682841751724e-05, 2.7531061277841218e-05, 3.236902557546273e-05, 3.1585503165842965e-05, 2.1721041775890626e-05, 1.926358527271077e-05, 1.579845957166981e-05, 1.4074928913032636e-05, 1.54751687659882e-05, 2.763647717074491e-05, 2.2666292352369055e-05, 1.9317671103635803e-05, 2.7111953386338428e-05, 4.116072886972688e-05, 4.391441325424239e-05, 5.37729101779405e-05, 8.704402716830373e-05, 0.0001490024442318827, 0.00010522329102968797, 7.839669706299901e-05, 4.157203875365667e-05, 4.663713843910955e-05, 2.6777688617585227e-05, 1.6396685168729164e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.579084118930041e-05, "first_to_last_ratio": 1.101662799461071, "imbalance_abs_log_ratio": 0.09682067430659903, "last_quartile_mean": 1.433364292324768e-05, "mean": 2.2897426177337366e-05, "normalized": [ 0.39886794777854795, 0.877049729540605, 0.6305457825207785, 0.5772129763101231, 0.4367770087210306, 0.8436499656131583, 0.972408138490726, 0.7805596072336106, 0.6568359938107564, 0.6066017752376727, 0.49315422395705855, 0.4249778682175203, 0.4986390140344086, 0.857095556009823, 0.6846582999905741, 0.5629018786115396, 0.7936422327048538, 1.202661838363613, 1.2736141330113293, 1.5255025761233132, 2.344832382358406, 4.19579128393446, 3.181636935928761, 2.172433813999188, 1.1390303293883177, 1.4232014401744595, 0.759652938746426, 0.4324960642291122, 0.4879317473364259, 0.30796555094004274, 0.21544260190277736, 0.24222836478058168 ], "peak_layer": 22, "peak_normalized": 4.19579128393446, "population_cv": 0.858855507029887, "rest_mean": 1.6084403240610198e-05, "spike_contrast": 3.7109085828539374, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.968775003566407e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.133049388765357e-06, 2.0082181436009705e-05, 1.4437875506700948e-05, 1.3216691513662226e-05, 1.0001069313148037e-05, 1.93174128071405e-05, 2.226564356533345e-05, 1.7872805983643048e-05, 1.5039853678899817e-05, 1.3889619367546402e-05, 1.1291962437098846e-05, 9.73089936451288e-06, 1.141755001299316e-05, 1.9625282220658846e-05, 1.5676912880735472e-05, 1.2889004210592248e-05, 1.8172364434576593e-05, 2.7537860660231672e-05, 2.9162485589040443e-05, 3.493008262012154e-05, 5.36906263732817e-05, 9.607282117940485e-05, 7.285129686351866e-05, 4.9743142881197855e-05, 2.6080862880917266e-05, 3.2587649911874905e-05, 1.7394097085343674e-05, 9.903046702675056e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.034897362245829e-05, "first_to_last_ratio": 1.0677221298454265, "imbalance_abs_log_ratio": 0.06552752863846958, "last_quartile_mean": 6.588696783182968e-05, "mean": 0.00010326150248829435, "normalized": [ 0.5211538788724873, 0.8562183017948943, 0.6056449715109208, 0.5367927676701035, 0.33719464941795635, 0.7400076028920698, 0.8752614309891937, 0.977887157814224, 0.5891182329173178, 0.4959165872409433, 0.4220040517626212, 0.3715438426401133, 0.3950700830815582, 0.7426266558595574, 0.622546917450552, 0.5356415681171627, 0.7637846157174611, 1.2023092054640232, 1.2927045115765263, 1.5352424858917262, 2.711590866084263, 4.42744668313384, 2.973944015958163, 2.3638740639156786, 1.2453125473146385, 1.2805717504661769, 0.7715376325621908, 0.48453681266643056, 0.47249674362438154, 0.27056316221994425, 0.2589436600974457, 0.3205125432754346 ], "peak_layer": 22, "peak_normalized": 4.42744668313384, "population_cv": 0.9059813522703718, "rest_mean": 6.990356949380064e-05, "spike_contrast": 4.054075388577877, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002833943406585604, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.381513255997561e-05, 8.841438830131665e-05, 6.25398097326979e-05, 5.5430027714464813e-05, 3.481922612991184e-05, 7.641429692739621e-05, 9.03808104339987e-05, 0.00010097809717990458, 6.083323387429118e-05, 5.120909190736711e-05, 4.357677244115621e-05, 3.8366175431292504e-05, 4.079553036717698e-05, 7.66847442719154e-05, 6.428513006540015e-05, 5.5311153118964285e-05, 7.886954699642956e-05, 0.00012415225501172245, 0.00013348661013878882, 0.0001585314457770437, 0.0002800029469653964, 0.00045718479668721557, 0.00030709392740391195, 0.00024409718753304332, 0.00012859284470323473, 0.00013223376299720258, 7.96701351646334e-05, 5.003399928682484e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.406091136388568e-06, "first_to_last_ratio": 0.13952873923075965, "imbalance_abs_log_ratio": 1.9694846829479884, "last_quartile_mean": 3.15783770474809e-05, "mean": 1.6862885999557875e-05, "normalized": [ 0.5744948983086973, 0.20976707009476936, 0.1380798037056023, 0.25679297795080097, 0.1911977573081938, 0.15397727234373496, 0.20783011320461262, 0.35817424975363493, 0.40845309421862647, 0.30185320868879156, 0.2734882217077138, 0.2725610842103788, 0.35283552344917235, 0.37101154440150896, 0.4205580365391262, 0.41850822128869175, 0.3521537888379847, 0.4454210973675298, 0.41314047811549354, 0.5286573606706721, 2.6023136902657806, 2.9811111131589443, 2.583962732113176, 2.2024122663803394, 2.8672118131663793, 2.8994466418579012, 2.457231935453014, 1.624795250078144, 1.8440638614795903, 1.4598750180798175, 0.9499358030010668, 0.8786840728001112 ], "peak_layer": 22, "peak_normalized": 2.9811111131589443, "population_cv": 0.9711949879643771, "rest_mean": 1.1718449412809503e-05, "spike_contrast": 3.8096203683057026, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.464284356799908e-05, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 9.687641977507155e-06, 3.5372781894693617e-06, 2.3284239887289004e-06, 4.330270712671336e-06, 3.2241459848592058e-06, 2.596501190055278e-06, 3.5046155062445905e-06, 6.039851541572716e-06, 6.88769796397537e-06, 5.090116246719845e-06, 4.611800704878988e-06, 4.596166490955511e-06, 5.949825208517723e-06, 6.2563253777625505e-06, 7.091822226357181e-06, 7.05725642546895e-06, 5.938329195487313e-06, 7.511085186706623e-06, 6.9667407842644025e-06, 8.914688805816695e-06, 4.388251909404062e-05, 5.027013685321435e-05, 4.357306897873059e-05, 3.7139026971999556e-05, 4.8349465942010283e-05, 4.88930381834507e-05, 4.1436022002017125e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.77339601350468e-05, "first_to_last_ratio": 0.14031202708197554, "imbalance_abs_log_ratio": 1.9638865715137788, "last_quartile_mean": 0.00012638945145226899, "mean": 6.748338435613732e-05, "normalized": [ 0.581715983352747, 0.21329408694826651, 0.1383654282103214, 0.26055913189981234, 0.18941324497814624, 0.15268445345034073, 0.20588194209370914, 0.3604059140126238, 0.4112482421872158, 0.3050474729826253, 0.2735060049358427, 0.27320166025083037, 0.35640352937895586, 0.3762475521803712, 0.424911703962158, 0.41981156521974744, 0.35303414718819204, 0.4487530978367584, 0.4155766160707765, 0.5338541629759799, 2.603208152672671, 2.9606390037219725, 2.573692405652612, 2.185365730719144, 2.873997131282303, 2.8916679373421434, 2.4621472618371403, 1.629468290031214, 1.8469331471633454, 1.452438714650874, 0.9530063641283487, 0.8735199206828114 ], "peak_layer": 22, "peak_normalized": 2.9606390037219725, "population_cv": 0.9681787165444765, "rest_mean": 4.699617262236576e-05, "spike_contrast": 3.7899751783986364, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017811432771850378, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.9256163290701807e-05, 1.4393806850421242e-05, 9.337367373518646e-06, 1.7583412045496516e-05, 1.278224681300344e-05, 1.03036636573961e-05, 1.3893610230297782e-05, 2.432141081953887e-05, 2.775242319330573e-05, 2.0585635866154917e-05, 1.8457110854797065e-05, 1.843657264544163e-05, 2.405131635896396e-05, 2.5390458176843822e-05, 2.8674479835899547e-05, 2.8330305212875828e-05, 2.382393904554192e-05, 3.028337778232526e-05, 2.8044516511727124e-05, 3.602628567023203e-05, 0.00017567329632584006, 0.00019979393982794136, 0.00017368147382512689, 0.00014747587556485087, 0.0001939470530487597, 0.00019513953884597868, 0.00016615403001196682, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015726652927696705, "first_to_last_ratio": 0.14031202298106044, "imbalance_abs_log_ratio": 1.9638866007408895, "last_quartile_mean": 0.11208343086764216, "mean": 0.05984494066797197, "normalized": [ 0.5817159394623853, 0.21329409008400638, 0.13836543030820245, 0.26055913499009653, 0.18941324063527437, 0.1526844532232541, 0.20588194088455028, 0.36040590002189726, 0.41124824181954295, 0.3050474835118139, 0.2735060178075916, 0.273201651110741, 0.35640353553867715, 0.37624753453424215, 0.42491172006038797, 0.41981156257536634, 0.35303415045149444, 0.4487531036157151, 0.4155766045421345, 0.5338542063524581, 2.6032081764056025, 2.9606389456555853, 2.5736924190660213, 2.185365706690064, 2.873996992351884, 2.891668001746537, 2.4621473625257404, 1.6294681803958442, 1.8469332209343858, 1.4524387488575696, 0.9530063959374594, 0.8735199079034743 ], "peak_layer": 22, "peak_normalized": 2.9606389456555853, "population_cv": 0.9681787143090018, "rest_mean": 0.04167667665967235, "spike_contrast": 3.789975088527889, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15795356631278992, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.03481275588274002, 0.01276457216590643, 0.008280470967292786, 0.015593145973980427, 0.011335424147546291, 0.009137392044067383, 0.012320992536842823, 0.02156846970319748, 0.024611126631498337, 0.018255548551678658, 0.016367951408028603, 0.016349736601114273, 0.021328948438167572, 0.022516511380672455, 0.02542881667613983, 0.0251235980540514, 0.021127307787537575, 0.026855602860450745, 0.02487015724182129, 0.03194847330451012, 0.1557888388633728, 0.1771792620420456, 0.1540224701166153, 0.1307830810546875, 0.1719941794872284, 0.17305169999599457, 0.14734706282615662, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0061295867558329e-06, "first_to_last_ratio": 0.11842320932790659, "imbalance_abs_log_ratio": 2.1334905510136277, "last_quartile_mean": 8.49605066832737e-06, "mean": 4.365163350783519e-06, "normalized": [ 0.5368020226986673, 0.16971060838105761, 0.13410619991648343, 0.2737672720355794, 0.17254528945791953, 0.13905042866169556, 0.15902329960072473, 0.2589205349972531, 0.33151883176756186, 0.2629812299966789, 0.24045074835041885, 0.24352968420582896, 0.32204843448988535, 0.36030617882440874, 0.42811660272336727, 0.41111875160131917, 0.35123001138976506, 0.43809006822116814, 0.40879993942951587, 0.3687203828633752, 2.5744168736304367, 2.993182569498261, 2.6417671760629977, 2.209152675699325, 2.926027285435564, 2.960732429505928, 2.7232927720785614, 1.7159123604866093, 1.8688437451101783, 1.456254776895987, 1.0064626168064281, 0.9131181991770502 ], "peak_layer": 22, "peak_normalized": 2.993182569498261, "population_cv": 1.010043630070163, "rest_mean": 3.0160778355484246e-06, "spike_contrast": 3.8627070547516635, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1650225133053027e-05, "top_five_layers": [ 22, 26, 25, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.3432285161106847e-06, 7.408145279441669e-07, 5.853954689882812e-07, 1.1950388625336927e-06, 7.531883738920442e-07, 6.069778351047717e-07, 6.941626793377509e-07, 1.1302304301352706e-06, 1.447133854526328e-06, 1.147956027125474e-06, 1.049606794367719e-06, 1.0630468523231684e-06, 1.4057940234124544e-06, 1.5727953268651618e-06, 1.8687989040699904e-06, 1.7946005073099514e-06, 1.5331763734138804e-06, 1.9123347101412946e-06, 1.784478513400245e-06, 1.6095247019620729e-06, 1.1237750186410267e-05, 1.306573085457785e-05, 1.1531745258253068e-05, 9.643312296248041e-06, 1.277258706977591e-05, 1.2924080692755524e-05, 1.188761780213099e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.197550119513835e-05, "first_to_last_ratio": 0.1587346511851631, "imbalance_abs_log_ratio": 1.8405213313314666, "last_quartile_mean": 7.544352229160722e-05, "mean": 4.3035416283032646e-05, "normalized": [ 0.6336653627326162, 0.22557503269458085, 0.11384714614846705, 0.24237874371328197, 0.21264877275758737, 0.17043176216342354, 0.2212795583484355, 0.40634029614109113, 0.4665961306783005, 0.344071525310092, 0.31263674643247324, 0.325989461953546, 0.4165298351149085, 0.4247779117423742, 0.4717272077470243, 0.4737094580054267, 0.40283409959357847, 0.521705164723695, 0.4749365612827932, 0.5725415698804192, 2.7082874349482955, 3.047902384226458, 2.6438885206938516, 2.1412461889756425, 2.9349243481759872, 2.7222959905080586, 2.283569880862303, 1.5378599969243498, 1.615015848446693, 1.3002882037476036, 0.8505045545457767, 0.7799943007808661 ], "peak_layer": 22, "peak_normalized": 3.047902384226458, "population_cv": 0.9449871736776022, "rest_mean": 2.9525086692619544e-05, "spike_contrast": 3.928564114944936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001159911960712634, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.727005266933702e-05, 9.707715435069986e-06, 4.899459327134537e-06, 1.0430870133859571e-05, 9.151428457698785e-06, 7.334601832553744e-06, 9.522857908450533e-06, 1.748702379700262e-05, 2.0080158719792962e-05, 1.4807261322857812e-05, 1.3454452528094407e-05, 1.4029092199052684e-05, 1.7925534848473035e-05, 1.8280494259670377e-05, 2.0300976757425815e-05, 2.038628372247331e-05, 1.733613316901028e-05, 2.245179894089233e-05, 2.0439092622837052e-05, 2.4639564799144864e-05, 0.00011655227717710659, 0.00013116774789523333, 0.00011378084309399128, 9.214942110702395e-05, 0.00012630569108296186, 0.00011715514119714499, 9.827438043430448e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 7.998359336625072e-06, "first_to_last_ratio": 0.19394161321534342, "imbalance_abs_log_ratio": 1.6401981280247118, "last_quartile_mean": 4.124106840208697e-05, "mean": 2.6285412580762113e-05, "normalized": [ 0.6933384670594045, 0.2456346256669455, 0.10397585233667712, 0.2374486790180797, 0.24369297059389394, 0.19793868329883177, 0.24449766268182652, 0.46778427650330523, 0.5456273930041772, 0.3924025527768945, 0.36245298346490396, 0.39321573934626614, 0.49513567631933314, 0.492561727340712, 0.5399874725375117, 0.5520792544136082, 0.47500593515532746, 0.6277771942942818, 0.5578424227128029, 0.6222043300599132, 2.928340543497061, 3.220069339518848, 2.7546600577445735, 2.05455257890152, 2.8717816218189083, 2.429966999218037, 1.946404647300736, 1.3608722565402098, 1.3761877892744612, 1.13757832466242, 0.7380055991332236, 0.6909763438053044 ], "peak_layer": 22, "peak_normalized": 3.220069339518848, "population_cv": 0.9165336328191448, "rest_mean": 1.7689689221461542e-05, "spike_contrast": 4.109869755811259, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.27023187209852e-05, "top_five_layers": [ 22, 21, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.822468766476959e-05, 6.4566074797767214e-06, 2.7330481771059567e-06, 6.241436494747177e-06, 6.405570275092032e-06, 5.2028999562026e-06, 6.426721938623814e-06, 1.2295902706682682e-05, 1.4342041140480433e-05, 1.0314462997484952e-05, 9.527226211503148e-06, 1.033583794196602e-05, 1.3014845535508357e-05, 1.2947188224643469e-05, 1.4193793504091445e-05, 1.4511630979541223e-05, 1.2485726983868517e-05, 1.6501382560818456e-05, 1.4663118236057926e-05, 1.6354897525161505e-05, 7.697263936279342e-05, 8.464085112791508e-05, 7.24073761375621e-05, 5.4004762205295265e-05, 7.548596477136016e-05, 6.387268513208255e-05, 5.116204920341261e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.53738644207624e-05, "first_to_last_ratio": 0.131222819995581, "imbalance_abs_log_ratio": 2.0308584847035727, "last_quartile_mean": 0.0002695709817999159, "mean": 0.00014080404997685037, "normalized": [ 0.577470254414939, 0.20295174191935292, 0.1151700578824124, 0.23719265118106814, 0.1794139596985285, 0.14982370308789797, 0.19985925174978317, 0.3479392297690331, 0.40530357048439714, 0.29950920312948914, 0.2746918219238327, 0.2680798564391957, 0.33955736615799725, 0.36796996435571633, 0.4082064803862017, 0.40241709302556344, 0.34764889743937344, 0.4446227376348821, 0.404675572121426, 0.502974600996141, 2.462986532813244, 2.8953303082983743, 2.595561209030766, 2.254551431946411, 3.008292708807131, 2.942171245358548, 2.6598889524780778, 1.7951160354833269, 1.7746811041390398, 1.3931209151914112, 0.9062455092896525, 0.836576033366786 ], "peak_layer": 25, "peak_normalized": 3.008292708807131, "population_cv": 0.9883689549295365, "rest_mean": 9.795413286526498e-05, "spike_contrast": 3.799672269993553, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003721936023794115, "top_five_layers": [ 25, 26, 22, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 8.131015056278557e-05, 2.8576427212101407e-05, 1.6216410585911945e-05, 3.339768591104075e-05, 2.5262212147936225e-05, 2.1095784177305177e-05, 2.814099207171239e-05, 4.8991252697305754e-05, 5.706838419428095e-05, 4.217210880597122e-05, 3.867772102239542e-05, 3.7746729503851384e-05, 4.7811052354518324e-05, 5.181166125112213e-05, 5.7477125665172935e-05, 5.666195647791028e-05, 4.8950372729450464e-05, 6.260468217078596e-05, 5.697995948139578e-05, 7.082086085574701e-05, 0.0003467984788585454, 0.0004076742334291339, 0.00036546553019434214, 0.00031744997249916196, 0.0004235797969158739, 0.00041426962707191706, 0.00037452313699759543, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } } }, "learned": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6127158914969186e-05, "first_to_last_ratio": 0.6403133863364645, "imbalance_abs_log_ratio": 0.4457975563250414, "last_quartile_mean": 2.518635290016391e-05, "mean": 2.1593947650444534e-05, "normalized": [ 0.6929919556166857, 0.8032314127318161, 0.7123629256452019, 0.6640523991876173, 0.8725736234454132, 0.8071370183700546, 0.7516577913014878, 0.6706888120378312, 0.5882487760705537, 0.5190639183458972, 0.4863138588077732, 0.4652918617004374, 0.6603036411218667, 0.6189718629251246, 0.5802646281311779, 0.5410038780854491, 0.6697462103535458, 0.6259079034957141, 0.6246747715872017, 0.6676260304525075, 3.1377243999857436, 2.6752853614525707, 2.136977079899559, 1.6970064982433113, 2.2713553531497497, 1.9850638882393974, 1.4820693911808296, 0.9615863897898643, 1.0163024380148542, 0.7191699586662397, 0.4720839007000451, 0.42326206126447913 ], "peak_layer": 21, "peak_normalized": 3.1377243999857436, "population_cv": 0.6918392109604528, "rest_mean": 1.6060819520875897e-05, "spike_contrast": 3.2048700555541787, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.147283955011517e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4964432011765894e-05, 1.7344937077723444e-05, 1.53827277245e-05, 1.4339512745209504e-05, 1.8842309145838954e-05, 1.7429274521418847e-05, 1.623125899641309e-05, 1.4482819096883759e-05, 1.2702613275905605e-05, 1.1208639079995919e-05, 1.0501436008780729e-05, 1.0047488103737123e-05, 1.4258562259783503e-05, 1.3366046005103271e-05, 1.2530204003269318e-05, 1.1682409422064666e-05, 1.446246460545808e-05, 1.3515822502085939e-05, 1.3489194316207431e-05, 1.4416681551665533e-05, 6.775585643481463e-05, 5.7769972045207396e-05, 4.61457711935509e-05, 3.664506948553026e-05, 4.9047528591472656e-05, 4.2865365685429424e-05, 3.2003728847485036e-05, 2.076444616250228e-05, 2.1945981643511914e-05, 1.5529718439211138e-05, 1.019415503833443e-05, 9.13989879336441e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.422064188882359e-05, "first_to_last_ratio": 0.6376814356469364, "imbalance_abs_log_ratio": 0.4499164375054873, "last_quartile_mean": 0.00010070959933727863, "mean": 8.629182207187114e-05, "normalized": [ 0.6940773401510585, 0.8062528427202595, 0.7180374505283312, 0.6647119372719499, 0.8588689882759897, 0.7941431696191276, 0.7473948010342778, 0.6703237246625169, 0.591781846497542, 0.522694550559361, 0.4893812082436792, 0.4670959071660659, 0.6619658672285874, 0.6164719403371215, 0.5787012511245688, 0.5393739243504363, 0.6707710740427385, 0.62668399370843, 0.6317390712162502, 0.6712190136492653, 3.138964807406113, 2.6795820619127526, 2.136435916928567, 1.6866743661161174, 2.264472697470994, 1.978410075097681, 1.4778942897678682, 0.9605324338748193, 1.022040338716437, 0.7239267286003249, 0.47869752189559756, 0.4306788598251707 ], "peak_layer": 21, "peak_normalized": 3.138964807406113, "population_cv": 0.6905024564046187, "rest_mean": 6.421987658271703e-05, "spike_contrast": 3.1996375366532934, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020548032771330326, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.989319834043272e-05, 6.957302684895694e-05, 6.196075992193073e-05, 5.7359204220119864e-05, 7.411336991935968e-05, 6.852806109236553e-05, 6.449405918829143e-05, 5.784345557913184e-05, 5.1065933803329244e-05, 4.5104265154805034e-05, 4.222959614708088e-05, 4.03065569116734e-05, 5.712224083254114e-05, 5.319648698787205e-05, 4.993718539481051e-05, 4.654355871025473e-05, 5.788205817225389e-05, 5.4077703680377454e-05, 5.4513915529241785e-05, 5.792071169707924e-05, 0.0002708669926505536, 0.00023122601851355284, 0.00018435694801155478, 0.00014554620429407805, 0.00019540547509677708, 0.0001707206101855263, 0.00012753019109368324, 8.288609387818724e-05, 8.81937230587937e-05, 6.246895645745099e-05, 4.1307681385660544e-05, 3.716406354214996e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05695150885730982, "first_to_last_ratio": 0.6376814663645546, "imbalance_abs_log_ratio": 0.4499163893346989, "last_quartile_mean": 0.08931027771905065, "mean": 0.07652445149142295, "normalized": [ 0.6940773558515339, 0.8062528453975966, 0.7180374361894372, 0.6647119756359035, 0.8588690718004696, 0.7941431432641803, 0.7473947840325739, 0.6703237318272384, 0.5917818305157142, 0.5226945671206367, 0.4893812028028422, 0.46709594515141906, 0.6619658291829611, 0.6164719318106706, 0.5787012336125282, 0.5393739103061749, 0.6707710619535627, 0.6266839981127393, 0.6317390378687008, 0.6712190249334776, 3.138964907679676, 2.679582133407049, 2.136436026253706, 1.6866743790728473, 2.2644726279739604, 1.9784099533155926, 1.4778942270811004, 0.9605324250805215, 1.0220403417906543, 0.7239267230206866, 0.4786975145327867, 0.43067882342105884 ], "peak_layer": 21, "peak_normalized": 3.138964907679676, "population_cv": 0.6905024682030625, "rest_mean": 0.05695082868138949, "spike_contrast": 3.1996376327558256, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.18222201466560364, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05311388894915581, 0.061698056757450104, 0.054947420954704285, 0.05086671933531761, 0.06572448462247849, 0.06077136844396591, 0.057193975895643234, 0.05129615589976311, 0.04528577998280525, 0.039998915046453476, 0.03744962811470032, 0.035744260996580124, 0.05065657198429108, 0.047175176441669464, 0.044284794479608536, 0.04127529263496399, 0.051330387592315674, 0.04795664921402931, 0.0483434833586216, 0.05136466771364212, 0.24020756781101227, 0.20505355298519135, 0.16348959505558014, 0.12907183170318604, 0.1732875257730484, 0.1513967365026474, 0.11309504508972168, 0.07350421696901321, 0.07821107655763626, 0.05539809539914131, 0.03663206472992897, 0.03295746073126793 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.592033183645981e-06, "first_to_last_ratio": 0.3776007228014791, "imbalance_abs_log_ratio": 0.9739179305137912, "last_quartile_mean": 6.864481520096888e-06, "mean": 4.875440396290287e-06, "normalized": [ 0.58913988046903, 0.5446521181917551, 0.5278862817384906, 0.53898932284961, 0.5219441801529001, 0.509889287242028, 0.5048315077449682, 0.5158761132703409, 0.4368296481828547, 0.4047199700448834, 0.3956716882892422, 0.40126275663148436, 0.5110508169296636, 0.5085656955747676, 0.5112234187675316, 0.511163584085558, 0.5662253891905324, 0.5872486754624854, 0.609842074156892, 0.6494088419115644, 3.025689231381945, 2.8686186674565652, 2.4420116557657345, 2.053486551003887, 2.698068277749679, 2.429082514259166, 1.8298867229318267, 1.2191127574146867, 1.182073363720965, 0.8592876866491284, 0.5581944369380665, 0.48806688384176716 ], "peak_layer": 21, "peak_normalized": 3.025689231381945, "population_cv": 0.8235639638153561, "rest_mean": 3.4149978226330994e-06, "spike_contrast": 3.736995148125518, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2761830294039101e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.87231637230434e-06, 2.6554189389571548e-06, 2.573678102635313e-06, 2.6278103177901357e-06, 2.5447077405260643e-06, 2.485934828655445e-06, 2.461275926179951e-06, 2.5151232421194436e-06, 2.1297369130479638e-06, 1.9731880911422195e-06, 1.92907373275375e-06, 1.9563326532079373e-06, 2.491597797416034e-06, 2.4794817363726906e-06, 2.4924393073888496e-06, 2.492147586963256e-06, 2.7605981358647114e-06, 2.863095915017766e-06, 2.973248683701968e-06, 3.166154101563734e-06, 1.4751567505300045e-05, 1.3985779332870152e-05, 1.1905882274731994e-05, 1.0011651284003165e-05, 1.3154271073290147e-05, 1.1842847015941516e-05, 8.92150364961708e-06, 5.943711585132405e-06, 5.763128228863934e-06, 4.18940589952399e-06, 2.7214437068323605e-06, 2.3795410015736707e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.8627718115312746e-05, "first_to_last_ratio": 0.778294860341054, "imbalance_abs_log_ratio": 0.2506498287376175, "last_quartile_mean": 6.247981400520075e-05, "mean": 6.0048676687074476e-05, "normalized": [ 0.768607661628769, 0.8690595287737466, 0.7879569251847977, 0.7385864958726717, 0.9274786983184008, 0.8617431831538573, 0.8054883103443636, 0.7195191288766225, 0.6496482220749353, 0.5799654886485536, 0.5429742744436054, 0.5176928240267351, 0.720980532018643, 0.6651070950014618, 0.6225467122437304, 0.5784265987882409, 0.7443177261701239, 0.697888823545442, 0.701818835999622, 0.7379727210931422, 3.1748914800668615, 2.647277084571918, 2.0581907003495648, 1.5579720703470805, 2.1379505239352077, 1.7650981274587072, 1.310550617537982, 0.8840683850126465, 0.8342765345805514, 0.5872947992080164, 0.42062138312406294, 0.3840285075999366 ], "peak_layer": 21, "peak_normalized": 3.1748914800668615, "population_cv": 0.6535432926484233, "rest_mean": 4.5422861014096134e-05, "spike_contrast": 3.0607513093024403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013902808132115752, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.615387297235429e-05, 5.218587466515601e-05, 4.7315770643763244e-05, 4.435114169609733e-05, 5.569386848947033e-05, 5.174653779249638e-05, 4.83685071230866e-05, 4.3206171540077776e-05, 3.901051604771055e-05, 3.482616011751816e-05, 3.2604886655462906e-05, 3.1086769013199955e-05, 4.329392686486244e-05, 3.993880091002211e-05, 3.738310624612495e-05, 3.473375181783922e-05, 4.4695294491248205e-05, 4.1907300328603014e-05, 4.214329237584025e-05, 4.431428533280268e-05, 0.00019064803200308233, 0.0001589654857525602, 0.00012359162792563438, 9.355416113976389e-05, 0.00012838109978474677, 0.00010599180677672848, 7.869683031458408e-05, 5.3087136620888487e-05, 5.009720189264044e-05, 3.52662755176425e-05, 2.525775744288694e-05, 2.3060403691488318e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.645518927442026e-05, "first_to_last_ratio": 0.9865830246987611, "imbalance_abs_log_ratio": 0.013507796190281324, "last_quartile_mean": 3.6950959383830195e-05, "mean": 4.1335310356771515e-05, "normalized": [ 0.8560051014711698, 0.9467981965967722, 0.8689980494059457, 0.8159268191754907, 1.0103083717536356, 0.9426554996877177, 0.8489560918930947, 0.765857482001633, 0.7201445334320749, 0.6498760503731635, 0.6109444686941845, 0.5791405787907885, 0.7989198470081217, 0.7291371869136885, 0.6790968935040115, 0.6275849613644364, 0.8454614229708973, 0.7893993011096712, 0.7856135784658201, 0.8181223517247423, 3.2763150867214206, 2.615409622894913, 1.9276349807221327, 1.3402369957405367, 1.940332562376331, 1.4667183642001942, 1.0945246384976166, 0.8103162680270223, 0.6598082696260246, 0.4694457579306494, 0.3666062775483438, 0.3437043893777548 ], "peak_layer": 21, "peak_normalized": 3.2763150867214206, "population_cv": 0.6207682663855968, "rest_mean": 3.199670040716969e-05, "spike_contrast": 2.867914594845514, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.176380408462136e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.5383236536290497e-05, 3.913619730155915e-05, 3.592030407162383e-05, 3.37265882990323e-05, 4.176141010248102e-05, 3.896495763910934e-05, 3.509186353767291e-05, 3.1656956707593054e-05, 2.9767397791147232e-05, 2.6862828235607594e-05, 2.5253579224226996e-05, 2.3938955564517528e-05, 3.3023599826265126e-05, 3.0139111913740635e-05, 2.8070680855307728e-05, 2.594141915324144e-05, 3.4947410313179716e-05, 3.263006510678679e-05, 3.2473581086378545e-05, 3.381734131835401e-05, 0.0001354275009362027, 0.00010810876847244799, 7.967939018271863e-05, 5.539911217056215e-05, 8.020424866117537e-05, 6.062725879019126e-05, 4.5242515625432134e-05, 3.349467442603782e-05, 2.7273379600956105e-05, 1.9404686099733226e-05, 1.5153784261201508e-05, 1.4207127605914138e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012982768839719938, "first_to_last_ratio": 0.6037915882942751, "imbalance_abs_log_ratio": 0.5045261930845532, "last_quartile_mean": 0.00021502069739653962, "mean": 0.0001790727353636612, "normalized": [ 0.6739765257680471, 0.7636067305383979, 0.68849644503889, 0.6447045606228033, 0.8477445590156324, 0.7818709864142286, 0.7391626168129651, 0.6604355769831363, 0.5800901925673246, 0.5130947085748045, 0.4841531977734674, 0.46136480942786895, 0.6373637447050293, 0.5885397553357715, 0.5617314689244236, 0.5244758601010534, 0.6356987553893274, 0.6101287890369049, 0.6118132813754859, 0.6643470301154188, 3.09815074914609, 2.693324715468958, 2.18351992306606, 1.7462447599227975, 2.367437166894702, 2.0445672235354038, 1.5627333414952298, 1.026078331181794, 1.036030398706582, 0.6947075889876938, 0.46018158540857673, 0.4142246216651318 ], "peak_layer": 21, "peak_normalized": 3.09815074914609, "population_cv": 0.7095514290759102, "rest_mean": 0.00013205833400004648, "spike_contrast": 3.278479211520481, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004329505027271807, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012069082004018128, 0.00013674114597961307, 0.00012329094170127064, 0.00011544900917215273, 0.00015180793707259, 0.00014001177623867989, 0.0001323638716712594, 0.00011826600530184805, 0.00010387833754066378, 9.188127296511084e-05, 8.669863746035844e-05, 8.261785842478275e-05, 0.00011413446918595582, 0.00010539142385823652, 0.00010059079068014398, 9.391932690050453e-05, 0.00011383631499484181, 0.00010925743117695674, 0.00010955907782772556, 0.00011896643991349265, 0.0005547943292185664, 0.0004823010240215808, 0.0003910088853444904, 0.0003127048257738352, 0.00042394344927743077, 0.0003661262453533709, 0.0002798429341055453, 0.0001837426534621045, 0.00018552479741629213, 0.0001244031882379204, 8.240597526310012e-05, 7.417633605655283e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5986858215910615e-05, "first_to_last_ratio": 0.8170320074925627, "imbalance_abs_log_ratio": 0.20207700803204537, "last_quartile_mean": 1.9566991340980167e-05, "mean": 1.9039267499465495e-05, "normalized": [ 0.7853354674160987, 0.9097466615865485, 0.8087937381555731, 0.7523818308832718, 0.9858416226009721, 0.9045115156777315, 0.8288509697382657, 0.741963942115696, 0.6090758738841542, 0.5628736284922787, 0.5329013290216021, 0.5131548852635935, 0.7265824288035061, 0.6757039396775449, 0.6225875966900762, 0.5898833165121294, 0.7532717273459274, 0.6927780667296713, 0.6897910451069404, 0.809449516707258, 3.207633782072864, 2.5002059817245676, 1.9824571594317655, 1.5924827370882133, 2.273874867822024, 1.6940739554643818, 1.0968135124011316, 0.8719494928225496, 0.815651324531277, 0.538892521487639, 0.48196570767333513, 0.44851985507141146 ], "peak_layer": 21, "peak_normalized": 3.207633782072864, "population_cv": 0.6428607369119902, "rest_mean": 1.4415789741582961e-05, "spike_contrast": 3.0526282764165082, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4006047392031175e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4952212040952872e-05, 1.7320910046692006e-05, 1.539884033263661e-05, 1.4324798939924221e-05, 1.8769702364807017e-05, 1.7221236703335308e-05, 1.578071533003822e-05, 1.4126449968898669e-05, 1.1596358490351122e-05, 1.0716701581259258e-05, 1.0146050954062957e-05, 9.770093129191082e-06, 1.3833597222401295e-05, 1.2864908057963476e-05, 1.18536117952317e-05, 1.1230946256546304e-05, 1.434174191672355e-05, 1.318998693022877e-05, 1.3133116226526909e-05, 1.541132587590255e-05, 6.107099761720747e-05, 4.760209048981778e-05, 3.77445321646519e-05, 3.0319704819703475e-05, 4.3292911868775263e-05, 3.225392720196396e-05, 2.088252585963346e-05, 1.660127963987179e-05, 1.5529403754044324e-05, 1.0260118870064616e-05, 9.176274033961818e-06, 8.539489499526098e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.365280069076107e-05, "first_to_last_ratio": 0.8123684077728273, "imbalance_abs_log_ratio": 0.2078013375593398, "last_quartile_mean": 7.83545988269907e-05, "mean": 7.614439766712167e-05, "normalized": [ 0.7854660713099145, 0.9130496523819581, 0.8147298517264512, 0.752236196131495, 0.9689924576782674, 0.8892579571020611, 0.8242433765322099, 0.7396130336387524, 0.6125904066287632, 0.5670744382794471, 0.5359292329408729, 0.5145351492524995, 0.7290151059417195, 0.673146260009013, 0.621425875046763, 0.5883337897938153, 0.7535116607120764, 0.6934400787203782, 0.6978162466203766, 0.8168476312813977, 3.2190719364475022, 2.5076839554965473, 1.973830744692153, 1.575947332799018, 2.2647184639164535, 1.6880790923847124, 1.0924366554795215, 0.8732736568641918, 0.8206255787733402, 0.5459821164389519, 0.48994786526283923, 0.4571481297165359 ], "peak_layer": 21, "peak_normalized": 3.2190719364475022, "population_cv": 0.642032487569562, "rest_mean": 5.76970003883751e-05, "spike_contrast": 3.0462648281411457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001757603429723531, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.9808840887853876e-05, 6.952361582079902e-05, 6.203711382113397e-05, 5.727857205783948e-05, 7.378334703389555e-05, 6.771201151423156e-05, 6.276151543715969e-05, 5.6317388953175396e-05, 4.664532752940431e-05, 4.3179541535209864e-05, 4.0808008634485304e-05, 3.917896901839413e-05, 5.551041613216512e-05, 5.125631651026197e-05, 4.731809895019978e-05, 4.479832205106504e-05, 5.7375691540073603e-05, 5.280157711240463e-05, 5.3134797781240195e-05, 6.219837086973712e-05, 0.00024511429364793, 0.00019094608433078974, 0.0001502961531514302, 0.00011999956041108817, 0.00017244562332052737, 0.00012853776570409536, 8.318293112097308e-05, 6.649489660048857e-05, 6.248604040592909e-05, 4.157347939326428e-05, 3.7306785088730976e-05, 3.480926898191683e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05644794087857008, "first_to_last_ratio": 0.8123684042388034, "imbalance_abs_log_ratio": 0.2078013419096122, "last_quartile_mean": 0.06948564294725657, "mean": 0.06752561370376498, "normalized": [ 0.7854660588804813, 0.9130496786513765, 0.814729886022376, 0.7522361645698925, 0.968992465622299, 0.8892579649660906, 0.8242433717205022, 0.7396130482968516, 0.6125904375263521, 0.5670744516239514, 0.535929266538159, 0.5145351229333333, 0.7290151143815181, 0.673146253269631, 0.6214258465577596, 0.5883338184574164, 0.7535116614810096, 0.6934400601144122, 0.6978162502950979, 0.8168476412095322, 3.21907174067581, 2.5076838612014067, 1.9738308079311102, 1.5759473804424433, 2.264718470873259, 1.6880792267982687, 1.0924366234670762, 0.8732736495232089, 0.8206255836175469, 0.5459820799959341, 0.4899478787351448, 0.45714813362074885 ], "peak_layer": 21, "peak_normalized": 3.21907174067581, "population_cv": 0.6420324679566953, "rest_mean": 0.05116627765474496, "spike_contrast": 3.0462647570380494, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15586602836847305, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.053039077669382095, 0.061654239892959595, 0.05501513555645943, 0.05079520866274834, 0.06543181091547012, 0.0600476898252964, 0.05565753951668739, 0.049942824989557266, 0.041365545243024826, 0.0382920503616333, 0.03618895262479782, 0.034744299948215485, 0.049227192997932434, 0.045454613864421844, 0.0419621616601944, 0.039727602154016495, 0.050881337374448776, 0.04682496562600136, 0.04712047055363655, 0.055158138275146484, 0.21736979484558105, 0.16933289170265198, 0.13328413665294647, 0.10641681402921677, 0.152926504611969, 0.11398858577013016, 0.07376745343208313, 0.05896833911538124, 0.055413246154785156, 0.03686777502298355, 0.03308403119444847, 0.03086920827627182 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.57929028180115e-06, "first_to_last_ratio": 0.4759084167214945, "imbalance_abs_log_ratio": 0.7425298451079264, "last_quartile_mean": 5.41971982670475e-06, "mean": 4.213722270662856e-06, "normalized": [ 0.6771451422985035, 0.6338625145334242, 0.6120949335724296, 0.6258725064817227, 0.6050121598754559, 0.5852465596314514, 0.5785033580390577, 0.5791973952882404, 0.4696864977471604, 0.4522091645384124, 0.4457992216966758, 0.45590579514105095, 0.5773211960070957, 0.5667516162949074, 0.5644132471304436, 0.568160519465189, 0.6503432281671621, 0.663170344529487, 0.6837490184222774, 0.7583004470800568, 3.021070651777523, 2.6274098317695387, 2.3058345630897463, 2.003282670865861, 2.819487095674051, 2.140274896548892, 1.4825743986898414, 1.122684410298403, 0.9855482185618173, 0.6429507763662341, 0.5708772581826982, 0.5252603622351902 ], "peak_layer": 21, "peak_normalized": 3.021070651777523, "population_cv": 0.7577432386283482, "rest_mean": 3.000000956658537e-06, "spike_contrast": 3.5892713108597136, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0767817366286181e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.8533015665743733e-06, 2.670920594027848e-06, 2.579198053354048e-06, 2.6372529191576177e-06, 2.549353212089045e-06, 2.4660664621478645e-06, 2.4376524834224256e-06, 2.4405769636359764e-06, 1.9791284557868494e-06, 1.9054838276133523e-06, 1.8784741087074508e-06, 1.921060402310104e-06, 2.4326711809408152e-06, 2.3881339075160213e-06, 2.378280669290689e-06, 2.394070634181844e-06, 2.7403657441027462e-06, 2.7944156499870587e-06, 2.881128466469818e-06, 3.1952674817148363e-06, 1.27299526866409e-05, 1.1071175322285853e-05, 9.71614645095542e-06, 8.441276804660447e-06, 1.1880535566888284e-05, 9.018523996928707e-06, 6.247156761673978e-06, 4.730680302600376e-06, 4.152826477366034e-06, 2.7092160053143743e-06, 2.4055182166193845e-06, 2.2133012862468604e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.826902750210138e-05, "first_to_last_ratio": 0.9723604213825404, "imbalance_abs_log_ratio": 0.028028739360968564, "last_quartile_mean": 4.964108620697516e-05, "mean": 5.395586174472555e-05, "normalized": [ 0.8546738942457572, 0.9667007478721396, 0.8780654782701063, 0.82151592085777, 1.0287569123437954, 0.9492731152891766, 0.8750422046820956, 0.7827885676521403, 0.6672237173045179, 0.6202571530123847, 0.5854234638750982, 0.5620770546607131, 0.7815896821357725, 0.715521927114241, 0.659703864497871, 0.6215694475292887, 0.8222045332968153, 0.7616919988241317, 0.764120987673305, 0.8753289635877315, 3.236716069528407, 2.479479992593944, 1.8829977352397056, 1.4470254883420661, 2.056662650357765, 1.5065673712263334, 1.0009483781315074, 0.8155586445002443, 0.6595748803016609, 0.47597993512775083, 0.43187920683965, 0.4130800130861142 ], "peak_layer": 21, "peak_normalized": 3.236716069528407, "population_cv": 0.610766278714797, "rest_mean": 4.176007456003895e-05, "spike_contrast": 2.8690828214345374, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001198131125420332, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.6114666474750265e-05, 5.2159171900711954e-05, 4.737677954835817e-05, 4.432559944689274e-05, 5.550746573135257e-05, 5.121884896652773e-05, 4.721365621662699e-05, 4.223603173159063e-05, 3.600063064368442e-05, 3.3466509194113314e-05, 3.158702747896314e-05, 3.032735185115598e-05, 4.217134483042173e-05, 3.860660217469558e-05, 3.5594890505308285e-05, 3.353731517563574e-05, 4.4362754124449566e-05, 4.109774818061851e-05, 4.122880636714399e-05, 4.722912854049355e-05, 0.00017463980475440621, 0.00013378247967921197, 0.00010159876546822488, 7.80755071900785e-05, 0.00011096900561824441, 8.128814079100266e-05, 5.400703230407089e-05, 4.400416946737096e-05, 3.558793105185032e-05, 2.568190757301636e-05, 2.3302414774661884e-05, 2.22880880755838e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.626883517426904e-05, "first_to_last_ratio": 1.1972713068748915, "imbalance_abs_log_ratio": 0.1800450565949769, "last_quartile_mean": 3.0292912697404972e-05, "mean": 3.794855416572318e-05, "normalized": [ 0.934241229967689, 1.0326674049103126, 0.9483113974161322, 0.8884949951731808, 1.1009339220259078, 1.0148817561286225, 0.9027362835592571, 0.8236285600155873, 0.7333418938739371, 0.6823440677494514, 0.6463725772705814, 0.6166567793546124, 0.850307320786337, 0.7701976529963521, 0.7042915982263429, 0.6605966097637871, 0.9142336932467707, 0.8480352948917127, 0.839474071242197, 0.9443144532969522, 3.2998505290664992, 2.4485943259044243, 1.7642227386757172, 1.2451697951023746, 1.763789999228952, 1.2716404306318911, 0.9094502638769789, 0.7804639985266583, 0.5161011792987255, 0.40746526774101827, 0.37633786588868356, 0.36085204416235356 ], "peak_layer": 21, "peak_normalized": 3.2998505290664992, "population_cv": 0.5857907121700514, "rest_mean": 3.01878957947742e-05, "spike_contrast": 2.6453022731935993, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.985610936884768e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.545310391928069e-05, 3.918823495041579e-05, 3.5987046430818737e-05, 3.3717100450303406e-05, 4.177885057288222e-05, 3.851329529425129e-05, 3.425753675401211e-05, 3.12555130221881e-05, 2.7829264581669122e-05, 2.5893970814649947e-05, 2.452890475979075e-05, 2.3401233192998916e-05, 3.2267933420371264e-05, 2.9227887353044935e-05, 2.672684786375612e-05, 2.5068686227314174e-05, 3.469384682830423e-05, 3.218171332264319e-05, 3.185682726325467e-05, 3.583536818041466e-05, 0.00012522455654107034, 9.292061440646648e-05, 6.694970215903595e-05, 4.72523934149649e-05, 6.693328032270074e-05, 4.825691576115787e-05, 3.4512322599766776e-05, 2.961748032248579e-05, 1.9585293557611294e-05, 1.546271778352093e-05, 1.4281477888289373e-05, 1.3693813343707006e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012846768913732376, "first_to_last_ratio": 0.7741677488258334, "imbalance_abs_log_ratio": 0.2559666991257704, "last_quartile_mean": 0.00016594296175753698, "mean": 0.0001574651118971815, "normalized": [ 0.7618335990450072, 0.8698560675479529, 0.7822046486498666, 0.7351395196601278, 0.9594102821252986, 0.8815927774487453, 0.812858049389013, 0.7238937569203876, 0.5978223149190788, 0.5606129559633132, 0.5319350341866579, 0.513003926829316, 0.6981154037320007, 0.6408130676433959, 0.6075107239537738, 0.5706379352682298, 0.7181374534396903, 0.6783796513052409, 0.6819318446184696, 0.7908903240994326, 3.2180960304949577, 2.5452212715171636, 2.028532306848687, 1.6608547003669625, 2.321647122642583, 1.7738587692821628, 1.1712480907534195, 0.9285924776215382, 0.7896984661161683, 0.5235677697382035, 0.4733567899608167, 0.4487468679123384 ], "peak_layer": 21, "peak_normalized": 3.2180960304949577, "population_cv": 0.6626251267593958, "rest_mean": 0.00011795681536939271, "spike_contrast": 3.1436073616095452, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00037080991314724087, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011996221292065457, 0.00013697198301088065, 0.00012316994252614677, 0.00011575882672332227, 0.0001510736474301666, 0.0001388201053487137, 0.0001279967837035656, 0.00011398801143513992, 9.413615771336481e-05, 8.82769818417728e-05, 8.376120968023315e-05, 8.078022074187174e-05, 0.00010992882016580552, 0.00010090570140164346, 9.566174412611872e-05, 8.985556632978842e-05, 0.0001130815944634378, 0.00010682112770155072, 0.00010738047421909869, 0.00012453763338271528, 0.0005067378515377641, 0.0004007835523225367, 0.0003194230666849762, 0.0002615266712382436, 0.0003655784239526838, 0.0002793208695948124, 0.00018443071166984737, 0.0001462209183955565, 0.00012434995733201504, 8.244365744758397e-05, 7.453717989847064e-05, 7.066197576932609e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6127134472299076e-05, "first_to_last_ratio": 0.6403082943802734, "imbalance_abs_log_ratio": 0.4458055086442437, "last_quartile_mean": 2.5186515017594502e-05, "mean": 2.159394537670778e-05, "normalized": [ 0.6929830574423667, 0.8032565154251953, 0.7123500282965459, 0.6640666207712503, 0.8725494553308608, 0.8071350816916083, 0.7516540798235798, 0.6706926732818747, 0.5882490907185235, 0.5190611932100045, 0.48631129869562834, 0.46529523801872363, 0.6602888008616835, 0.6189890701429912, 0.5802627939183058, 0.5409897412703016, 0.6697392892795451, 0.6259125602673482, 0.6246778698614331, 0.6676379780381497, 3.137658015393984, 2.675256160517513, 2.137000554072058, 1.6970284098395438, 2.2713857488305376, 1.9850977916906365, 1.4820653354312663, 0.9615821107638199, 1.0163133272453195, 0.7191715506408339, 0.4720786856528992, 0.423259873575668 ], "peak_layer": 21, "peak_normalized": 3.137658015393984, "population_cv": 0.691835904675886, "rest_mean": 1.6060833668571258e-05, "spike_contrast": 3.20486156963133, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.147274860064499e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4964238289394416e-05, 1.7345477317576297e-05, 1.5382447600131854e-05, 1.43398183354293e-05, 1.8841785276890732e-05, 1.7429230865673162e-05, 1.623117714188993e-05, 1.4482900951406918e-05, 1.2702618732873816e-05, 1.1208579053345602e-05, 1.0501379620109219e-05, 1.0047559953818563e-05, 1.4258240298659075e-05, 1.3366416169446893e-05, 1.2530163076007739e-05, 1.1682102922350168e-05, 1.4462313629337586e-05, 1.3515921637008432e-05, 1.3489259799825959e-05, 1.4416938029171433e-05, 6.775441579520702e-05, 5.7769335398916155e-05, 4.614627323462628e-05, 3.664553878479637e-05, 4.9048179789679125e-05, 4.286609328119084e-05, 3.200363789801486e-05, 2.0764351575053297e-05, 2.1946214474155568e-05, 1.55297511810204e-05, 1.0194041351496708e-05, 9.139850590145215e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.422088154067751e-05, "first_to_last_ratio": 0.637682021407296, "imbalance_abs_log_ratio": 0.44991551892756126, "last_quartile_mean": 0.00010070988264487823, "mean": 8.629204910448607e-05, "normalized": [ 0.6940783808537224, 0.8062729813904731, 0.7180303336815479, 0.6647219507645815, 0.8588589713696609, 0.794136864353616, 0.7473889560340541, 0.6703283692089702, 0.5917748510321142, 0.5226947352403994, 0.4893764215021349, 0.4670989784552037, 0.661952700545094, 0.6164859593679979, 0.5786990118719374, 0.539368710966703, 0.6707742418503443, 0.6266861813765591, 0.6317438172781369, 0.6712283354793382, 3.1389231589965445, 2.679577878793134, 2.1364533990925567, 1.686690164787602, 2.264487313238182, 1.9784244316759128, 1.477878596963198, 0.9605191983702851, 1.022047261980904, 0.7239260887312458, 0.4786954192739982, 0.4306763354738471 ], "peak_layer": 21, "peak_normalized": 3.1389231589965445, "population_cv": 0.690500620274875, "rest_mean": 6.42200389443638e-05, "spike_contrast": 3.1996384204494492, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002054809039691463, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.98934457229916e-05, 6.957494770176709e-05, 6.196030881255865e-05, 5.736021921620704e-05, 7.411270053125918e-05, 6.852769729448482e-05, 6.449372449424118e-05, 5.784400855191052e-05, 5.106546450406313e-05, 4.51043997600209e-05, 4.2229294194839895e-05, 4.030692798551172e-05, 5.7121254940284416e-05, 5.319783667800948e-05, 4.993712354917079e-05, 4.654323129216209e-05, 5.788248381577432e-05, 5.40780347364489e-05, 5.451446850202046e-05, 5.7921668485505506e-05, 0.00027086411137133837, 0.00023122626589611173, 0.00018435894162394106, 0.00014554795052390546, 0.00019540725043043494, 0.0001707222982076928, 0.00012752917245961726, 8.288516983157024e-05, 8.819455251796171e-05, 6.24690655968152e-05, 4.13076086260844e-05, 3.716394348884933e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.056951718870550394, "first_to_last_ratio": 0.6376820191187264, "imbalance_abs_log_ratio": 0.4499155225164498, "last_quartile_mean": 0.08931052964180708, "mean": 0.07652465114369988, "normalized": [ 0.6940783685018588, 0.8062729890693022, 0.7180303052956253, 0.6647219248274491, 0.8588589447083164, 0.7941368847913186, 0.7473889396113633, 0.6703284088385635, 0.5917748342982165, 0.5226947125226558, 0.4893764209845987, 0.46709901042220126, 0.6619527107838585, 0.616485998695461, 0.5786990422522198, 0.5393687059717984, 0.6707742773702546, 0.6266862088911475, 0.6317438155472213, 0.671228324296238, 3.138923030946758, 2.6795778685275287, 2.136453429700619, 1.6866902298133055, 2.2644871659665227, 1.9784244587489672, 1.4778786878450325, 0.9605192092561265, 1.0220472167545944, 0.7239261000048334, 0.4786954380387232, 0.4306763367173204 ], "peak_layer": 21, "peak_normalized": 3.138923030946758, "population_cv": 0.6905006031140464, "rest_mean": 0.056950972587973984, "spike_contrast": 3.199638339154521, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.18222251534461975, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05311410501599312, 0.0616997592151165, 0.05494701862335205, 0.05086761340498924, 0.06572388112545013, 0.060771048069000244, 0.05719367787241936, 0.051296647638082504, 0.045285362750291824, 0.03999903053045273, 0.03744935989379883, 0.03574458882212639, 0.05065570026636124, 0.04717637598514557, 0.04428474232554436, 0.04127500206232071, 0.05133076757192612, 0.04795694351196289, 0.048343975096940994, 0.05136551335453987, 0.24020498991012573, 0.20505376160144806, 0.163491353392601, 0.1290733814239502, 0.17328909039497375, 0.15139824151992798, 0.11309415102005005, 0.07350339740514755, 0.07821180671453476, 0.05539819225668907, 0.036632001399993896, 0.03295735642313957 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.591977136034984e-06, "first_to_last_ratio": 0.377584641115206, "imbalance_abs_log_ratio": 0.9739605205507381, "last_quartile_mean": 6.864625447633443e-06, "mean": 4.875532418679995e-06, "normalized": [ 0.5891470886653372, 0.5445974877366685, 0.5278757119769764, 0.5390247128297342, 0.5219320902964267, 0.5098223481982903, 0.5047854170324074, 0.5158515929518226, 0.43682532071303765, 0.4047462819890774, 0.3956760190748421, 0.40124851416781243, 0.5110306315304836, 0.5086020328656453, 0.5111981468235955, 0.5112038363741968, 0.5662160078662817, 0.5872378713533353, 0.609816433193417, 0.6494206487580901, 3.0257058078799917, 2.8686033250815077, 2.442085511360172, 2.0535509509100836, 2.6979608311079244, 2.429139638501957, 1.8298842703872822, 1.2190711864897739, 1.1821681083908706, 0.8593060718331675, 0.5582216762792698, 0.48804442738052095 ], "peak_layer": 21, "peak_normalized": 3.0257058078799917, "population_cv": 0.8235699242655958, "rest_mean": 3.415056493462611e-06, "spike_contrast": 3.7370106290435205, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2762102414853871e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.872405730158789e-06, 2.6552027065918082e-06, 2.573675146777532e-06, 2.6280324618710438e-06, 2.544696826589643e-06, 2.4856553864083253e-06, 2.4610976652184036e-06, 2.515051164664328e-06, 2.129756012436701e-06, 1.973353619177942e-06, 1.929131258293637e-06, 1.956300138772349e-06, 2.491546410965384e-06, 2.4797056994430022e-06, 2.492363137207576e-06, 2.49239087679598e-06, 2.7606045023276238e-06, 2.8630972792598186e-06, 2.9731797894783085e-06, 3.1662714263802627e-06, 1.4751926755707245e-05, 1.398596850776812e-05, 1.1906467079825234e-05, 1.0012154234573245e-05, 1.315399549639551e-05, 1.1843349057016894e-05, 8.921660082705785e-06, 5.943621090409579e-06, 5.7636989367892966e-06, 4.189574610791169e-06, 2.7216278795094695e-06, 2.3794764274498448e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.862782361669815e-05, "first_to_last_ratio": 0.7782948608369358, "imbalance_abs_log_ratio": 0.25064982810047876, "last_quartile_mean": 6.247994951991132e-05, "mean": 6.004880788168521e-05, "normalized": [ 0.7686076787185943, 0.8690658694356951, 0.7879558094938062, 0.7385895471539009, 0.9274752179545845, 0.8617380288980451, 0.805485823509466, 0.7195218583140428, 0.6496432282856196, 0.579963373368429, 0.542973633406911, 0.5176967820022496, 0.7209744736270646, 0.6651136389238139, 0.6225458367735092, 0.578423456947426, 0.7443181598307745, 0.6978871776308329, 0.7018192413444355, 0.7379772277240845, 3.17488720924795, 2.647274451154299, 2.058197351015695, 1.557976178868823, 2.137956515672529, 1.7650930593990968, 1.3105561147977765, 0.8840594863751997, 0.8342799220515003, 0.5872955759322932, 0.4206210094039151, 0.3840270627376372 ], "peak_layer": 21, "peak_normalized": 3.17488720924795, "population_cv": 0.6535433957622422, "rest_mean": 4.5422938354830774e-05, "spike_contrast": 3.060755388404179, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013902850332669914, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.615397483576089e-05, 5.218636943027377e-05, 4.7315807023551315e-05, 4.4351421820465475e-05, 5.569378117797896e-05, 5.17463413416408e-05, 4.8368463467340916e-05, 4.320642983657308e-05, 3.9010301406960934e-05, 3.482610918581486e-05, 3.260491939727217e-05, 3.108707460341975e-05, 4.329365765443072e-05, 3.9939281123224646e-05, 3.738313534995541e-05, 3.47336390404962e-05, 4.4695418182527646e-05, 4.19072930526454e-05, 4.2143408791162074e-05, 4.43146527686622e-05, 0.00019064819207414985, 0.00015896567492745817, 0.0001235922973137349, 9.355461224913597e-05, 0.00012838174006901681, 0.00010599173401715234, 7.869733235565946e-05, 5.308671825332567e-05, 5.009751475881785e-05, 3.526639920892194e-05, 2.5257790184696205e-05, 2.3060367311700247e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.645344395408756e-05, "first_to_last_ratio": 0.9865245275167812, "imbalance_abs_log_ratio": 0.01356709065898008, "last_quartile_mean": 3.695138127568498e-05, "mean": 4.1334421752026174e-05, "normalized": [ 0.8559009893021382, 0.9469134291395908, 0.8691128415923758, 0.8158459610499553, 1.0100395593992384, 0.9426509450849102, 0.8490005706596633, 0.7658551995239993, 0.720151565776352, 0.6499584076998435, 0.6111014164691825, 0.5791973878203037, 0.798669989787535, 0.7292203680582068, 0.6791147051708346, 0.6276623067938342, 0.8453470506184937, 0.7894265691211152, 0.7857081832788244, 0.8180697050541997, 3.276793902438668, 2.6146895715088023, 1.9277214837041228, 1.3401559674726915, 1.9403857172500238, 1.4668533992566692, 1.0945373428822949, 0.8102509556533821, 0.6598620161196013, 0.46946557551492035, 0.3666267887349235, 0.3437101280633033 ], "peak_layer": 21, "peak_normalized": 3.276793902438668, "population_cv": 0.6207652283001258, "rest_mean": 3.1996292111418574e-05, "spike_contrast": 2.867842357851226, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.176032181130722e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.5378172469791025e-05, 3.9140119042713195e-05, 3.5924276744481176e-05, 3.372252103872597e-05, 4.174940113443881e-05, 3.8963931729085743e-05, 3.509294765535742e-05, 3.165618181810714e-05, 2.976704854518175e-05, 2.6865654945140705e-05, 2.5259523681597784e-05, 2.3940789105836302e-05, 3.301256219856441e-05, 3.0141902243485674e-05, 2.8070813641534187e-05, 2.5944058506865986e-05, 3.4941931517096236e-05, 3.263049075030722e-05, 3.2476793421665207e-05, 3.381443821126595e-05, 0.0001354443811578676, 0.00010807668149936944, 7.968125282786787e-05, 5.53945719730109e-05, 8.020472159842029e-05, 6.06315370532684e-05, 4.524206815403886e-05, 3.349125472595915e-05, 2.7275014872429892e-05, 1.940508809639141e-05, 1.5154306311160326e-05, 1.4207059393811505e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012982848875253694, "first_to_last_ratio": 0.6036924070795879, "imbalance_abs_log_ratio": 0.5046904705661434, "last_quartile_mean": 0.00021505734912352636, "mean": 0.00017907720302900998, "normalized": [ 0.6739415088564852, 0.7635979187076084, 0.6884132846665824, 0.6448880523181134, 0.847636297931654, 0.7820437425942102, 0.7390348805248977, 0.6603333703627584, 0.580097091853396, 0.5130765445550399, 0.4840585582545864, 0.4614791311662735, 0.637447225260648, 0.5884103323685723, 0.5619029319457929, 0.5245290839622635, 0.6356966288449003, 0.6100272686595811, 0.6118819192326126, 0.6643605628507145, 3.097964891544288, 2.6928767346712488, 2.1827809090098653, 1.7461631642015993, 2.367937988760414, 2.0449314566347367, 1.563314534665903, 1.0260809297257236, 1.0359670091642164, 0.6947146354097079, 0.460142110411202, 0.41426930088440395 ], "peak_layer": 21, "peak_normalized": 3.097964891544288, "population_cv": 0.7095182139952283, "rest_mean": 0.00013206795362445214, "spike_contrast": 3.278063587209748, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004329271498136222, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012068756041117013, 0.00013674297952093184, 0.0001232791255461052, 0.0001154847486759536, 0.00015179233741946518, 0.0001400462060701102, 0.00013234429934527725, 0.00011825065303128213, 0.00010388216469436884, 9.188031253870577e-05, 8.668385271448642e-05, 8.264039206551388e-05, 0.00011415226617828012, 0.00010537087655393407, 0.00010062400542665273, 9.393120126333088e-05, 0.00011383877426851541, 0.00010924197704298422, 0.00010957410268019885, 0.00011897183139808476, 0.0005547748878598213, 0.0004822328337468207, 0.00039088630001060665, 0.0003126980154775083, 0.0004240437119733542, 0.00036620060564018786, 0.0002799539943225682, 0.00018374770297668874, 0.00018551807443145663, 0.0001244075538124889, 8.240096212830395e-05, 7.418618770316243e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5986864127626177e-05, "first_to_last_ratio": 0.8170274391577745, "imbalance_abs_log_ratio": 0.20208259942567244, "last_quartile_mean": 1.956710798367567e-05, "mean": 1.9039307829871177e-05, "normalized": [ 0.7853167979867789, 0.9097665173048055, 0.808777694116954, 0.752402497635975, 0.9858540561935626, 0.9044938358034963, 0.8288447236919866, 0.7419578801196516, 0.6090850929460484, 0.5628647453138417, 0.5328979550345672, 0.5131403750850531, 0.7265824660927316, 0.6757007408890954, 0.6225826951824261, 0.5898791530514104, 0.7532772493405897, 0.6927950381940383, 0.6897848547806317, 0.8094505726944553, 3.2076147584918284, 2.5002121502474965, 1.982484105644761, 1.5924612114390058, 2.273889158865966, 1.6940625327896233, 1.0968336406315982, 0.8719471681044353, 0.815653609383569, 0.5388945327422838, 0.4819664542042268, 0.448525735997107 ], "peak_layer": 21, "peak_normalized": 3.2076147584918284, "population_cv": 0.6428608089466793, "rest_mean": 1.4415815443229534e-05, "spike_contrast": 3.052631111367605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.400616671773605e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4951888260839041e-05, 1.7321324776276015e-05, 1.5398567484226078e-05, 1.4325222764455248e-05, 1.8769978851196356e-05, 1.7220936570083722e-05, 1.5780629837536253e-05, 1.4126364476396702e-05, 1.1596558579185512e-05, 1.0716555152612273e-05, 1.0146008207811974e-05, 9.769837561179884e-06, 1.3833627235726453e-05, 1.286487440665951e-05, 1.1853543583129067e-05, 1.1230890777369495e-05, 1.4341877431434114e-05, 1.3190337995183654e-05, 1.3133026186551433e-05, 1.541137862659525e-05, 6.107076478656381e-05, 4.760230876854621e-05, 3.7745125155197456e-05, 3.03193592117168e-05, 4.329327566665597e-05, 3.225377804483287e-05, 2.0882953322143294e-05, 1.6601270544924773e-05, 1.5529480151599273e-05, 1.0260178896714933e-05, 9.176307685265783e-06, 8.539619557268452e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.365272747643758e-05, "first_to_last_ratio": 0.8123633197064085, "imbalance_abs_log_ratio": 0.2078076008289402, "last_quartile_mean": 7.835499945940683e-05, "mean": 7.614457786075945e-05, "normalized": [ 0.7854618236676242, 0.9130540849409994, 0.8147131127464502, 0.7522410092502885, 0.9690072688462938, 0.8892440039434748, 0.8242354060534288, 0.7396083689563772, 0.6126026690255093, 0.5670644008562938, 0.5359283946772616, 0.5145269561429746, 0.7290174418180176, 0.6731434248219464, 0.6214223500409742, 0.5883307730946398, 0.7535191463387267, 0.6934458909688136, 0.6978155985835179, 0.8168473226649812, 3.219066994145898, 2.5077095541323673, 1.973834673592298, 1.5759251613516239, 2.2647260999477155, 1.6880724220748888, 1.092454232265147, 0.8732751258075867, 0.8206258345435784, 0.5459870832104624, 0.48994656248557816, 0.45714680900426247 ], "peak_layer": 21, "peak_normalized": 3.219066994145898, "population_cv": 0.6420341358971389, "rest_mean": 5.7697108584559625e-05, "spike_contrast": 3.04626897714722, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017576091195223854, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.980865898891352e-05, 6.95241178618744e-05, 6.203598604770377e-05, 5.727907409891486e-05, 7.378464943030849e-05, 6.77111092954874e-05, 6.276105705183e-05, 5.631716703646816e-05, 4.664637162932195e-05, 4.3178879423066974e-05, 4.080804137629457e-05, 3.917843787348829e-05, 5.551072536036372e-05, 5.1256221922812983e-05, 4.731794251711108e-05, 4.47981983597856e-05, 5.737639730796218e-05, 5.2802144637098536e-05, 5.3134874178795144e-05, 6.219849456101656e-05, 0.0002451144973747432, 0.00019094848539680243, 0.00015029680798761547, 0.00011999815615126863, 0.0001724466128507629, 0.00012853756197728217, 8.318446634802967e-05, 6.64951658109203e-05, 6.248620775295421e-05, 4.157395596848801e-05, 3.7306774174794555e-05, 3.4809250792022794e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05644787335768342, "first_to_last_ratio": 0.8123632868395458, "imbalance_abs_log_ratio": 0.2078076412872703, "last_quartile_mean": 0.06948599754832685, "mean": 0.06752577290171757, "normalized": [ 0.7854618348329956, 0.9130540359301327, 0.8147131249195937, 0.7522410113175043, 0.9690071730110875, 0.88924395208314, 0.8242353599678512, 0.7396083806676593, 0.6126026750590019, 0.5670643980854334, 0.5359283892166958, 0.5145269586503932, 0.7290174229589911, 0.6731434525628229, 0.6214223402600404, 0.5883307763560162, 0.7535191533057549, 0.6934459281737582, 0.6978155981634587, 0.8168473704719704, 3.2190670201887848, 2.507709505453073, 1.9738347607250095, 1.5759251284181852, 2.264726151347608, 1.68807248858185, 1.0924542395938426, 0.8732750663142743, 0.820625855657368, 0.5459870819951741, 0.48994655813463567, 0.45714680759589343 ], "peak_layer": 21, "peak_normalized": 3.2190670201887848, "population_cv": 0.6420341436220799, "rest_mean": 0.05116637251167386, "spike_contrast": 3.0462690113979045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15586653500795364, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05303891748189926, 0.06165467947721481, 0.05501413345336914, 0.05079565569758415, 0.06543295830488205, 0.060046885162591934, 0.05565712973475456, 0.04994262754917145, 0.041366469115018845, 0.038291461765766144, 0.03618897870182991, 0.03474383056163788, 0.04922746494412422, 0.04545453190803528, 0.041962023824453354, 0.03972749039530754, 0.05088196322321892, 0.04682547226548195, 0.04712053760886192, 0.05515825003385544, 0.21736998856067657, 0.1693350225687027, 0.13328471779823303, 0.10641556233167648, 0.15292738378047943, 0.11398839950561523, 0.07376881688833237, 0.058968573808670044, 0.055413395166397095, 0.036868199706077576, 0.033084020018577576, 0.030869191512465477 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.579253447265728e-06, "first_to_last_ratio": 0.47590496457778414, "imbalance_abs_log_ratio": 0.7425370989326324, "last_quartile_mean": 5.419681741614113e-06, "mean": 4.2137082871818166e-06, "normalized": [ 0.6771144196047112, 0.6338426021784607, 0.6121020910961615, 0.6258737201152189, 0.6050362914422536, 0.5852204423685657, 0.5784631886764509, 0.5792281322851218, 0.4697078599262646, 0.4522117174548025, 0.44580914592548354, 0.4558904724294877, 0.5773213311917524, 0.5667240346830847, 0.5644321177223985, 0.5681434108598962, 0.6503548834184826, 0.6631947770191623, 0.6837887360559592, 0.7582966501774209, 3.020973835679826, 2.627473806530212, 2.3058644468737186, 2.00331262981993, 2.819406877957528, 2.1402761714678284, 1.4825745701937771, 1.1227288221975413, 0.9855094000056028, 0.6429466506282437, 0.5709200007532207, 0.5252567632614316 ], "peak_layer": 21, "peak_normalized": 3.020973835679826, "population_cv": 0.7577358346351384, "rest_mean": 2.999999306094078e-06, "spike_contrast": 3.589246425218318, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0767736785055605e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.8531626412586775e-06, 2.670827825568267e-06, 2.5792196538532153e-06, 2.6372492811788106e-06, 2.5494464352959767e-06, 2.465948227836634e-06, 2.43747513195558e-06, 2.440698381178663e-06, 1.979211901925737e-06, 1.9054882614000235e-06, 1.8785096926876577e-06, 1.920989461723366e-06, 2.4326636776095256e-06, 2.3880097614892293e-06, 2.3783522919984534e-06, 2.393990598648088e-06, 2.7404057618696243e-06, 2.7945093279413413e-06, 2.8812862638005754e-06, 3.1952408789948095e-06, 1.2729502486763522e-05, 1.1071408152929507e-05, 9.716240128909703e-06, 8.441375030088238e-06, 1.1880158126587048e-05, 9.018499440571759e-06, 6.247136752790539e-06, 4.7308517423516605e-06, 4.152649125899188e-06, 2.709189629968023e-06, 2.405690338491695e-06, 2.2132787762529915e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.826905797017389e-05, "first_to_last_ratio": 0.9723593605402814, "imbalance_abs_log_ratio": 0.028029830358518852, "last_quartile_mean": 4.964117169947713e-05, "mean": 5.395593586854375e-05, "normalized": [ 0.8546740011831107, 0.9667023865336807, 0.8780594173977533, 0.8215201862718362, 1.028762848378348, 0.9492707323927138, 0.8750383055629682, 0.7827836490446655, 0.6672270484578459, 0.6202534690629704, 0.5854228619045188, 0.5620710233383177, 0.781592991024926, 0.7155185842665527, 0.6597031604820425, 0.621567177701999, 0.8222052916650392, 0.7616961441483728, 0.7641199379358804, 0.8753280307745216, 3.236716477578425, 2.479496544122304, 1.8830004075567146, 1.4470156791415152, 2.05665793704697, 1.5065644924277526, 1.0009572516438536, 0.8155596816984735, 0.6595753226879756, 0.4759799891964506, 0.4318790517932509, 0.413079917578251 ], "peak_layer": 21, "peak_normalized": 3.236716477578425, "population_cv": 0.6107674157639074, "rest_mean": 4.176012171902349e-05, "spike_contrast": 2.869084843241085, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011981333227595314, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.61147355963476e-05, 5.2159331971779466e-05, 4.737651761388406e-05, 4.4325890485197306e-05, 5.550786227104254e-05, 5.121879075886682e-05, 4.72135106974747e-05, 4.2235824366798624e-05, 3.6000859836349264e-05, 3.3466356399003416e-05, 3.158703839289956e-05, 3.0327068088809028e-05, 4.217158129904419e-05, 3.860647484543733e-05, 3.559490141924471e-05, 3.353723877808079e-05, 4.4362855987856165e-05, 4.1098028304986656e-05, 4.122880636714399e-05, 4.7229143092408776e-05, 0.00017464006668888032, 0.00013378355652093887, 0.00010159904923057184, 7.807508518453687e-05, 0.00011096890375483781, 8.128809713525698e-05, 5.400758527684957e-05, 4.4004285882692784e-05, 3.558800381142646e-05, 2.5681945771793835e-05, 2.330243842152413e-05, 2.228811354143545e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.626825309765991e-05, "first_to_last_ratio": 1.1971952602722011, "imbalance_abs_log_ratio": 0.17998153797771219, "last_quartile_mean": 3.0294350722215313e-05, "mean": 3.794959516767449e-05, "normalized": [ 0.9342317076920968, 1.032527492584725, 0.9483990781630639, 0.8885231558783968, 1.1008949985426397, 1.0148066560537, 0.9027782413985963, 0.8234017777861798, 0.7333936750261243, 0.6823073758552121, 0.6464481216407498, 0.616777331910669, 0.8501710687237772, 0.7702178906455867, 0.7042577074377577, 0.660638930723233, 0.914060026408416, 0.847914539167618, 0.8394293783449699, 0.9442746494437388, 3.300220154812763, 2.4489522165880904, 1.7640184700183157, 1.245126339879113, 1.7643693302152461, 1.271277790934411, 0.9095623054467832, 0.780451313056054, 0.5159478762516835, 0.40747033934370974, 0.37633772798986104, 0.36081233203671814 ], "peak_layer": 21, "peak_normalized": 3.300220154812763, "population_cv": 0.5858778602410362, "rest_mean": 3.018723526582579e-05, "spike_contrast": 2.6456990159702416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.986633863765746e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.5453715099720284e-05, 3.9184000343084335e-05, 3.599136107368395e-05, 3.371909406268969e-05, 4.1778519516810775e-05, 3.85115017706994e-05, 3.4260068787261844e-05, 3.124776412732899e-05, 2.783199306577444e-05, 2.5893288693623617e-05, 2.453244451317005e-05, 2.3406450054608285e-05, 3.226364788133651e-05, 2.9229457140900195e-05, 2.6726294890977442e-05, 2.5070979972952046e-05, 3.468820796115324e-05, 3.217801349819638e-05, 3.185600508004427e-05, 3.5834840673487633e-05, 0.00012524201883934438, 9.293674520449713e-05, 6.694378680549562e-05, 4.725204053102061e-05, 6.695710180792958e-05, 4.8244477511616424e-05, 3.451752127148211e-05, 2.9617811378557235e-05, 1.95800130313728e-05, 1.546333442092873e-05, 1.4281864423537627e-05, 1.3692681932298001e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012846494701079791, "first_to_last_ratio": 0.7741545804954827, "imbalance_abs_log_ratio": 0.25598370893064326, "last_quartile_mean": 0.00016594224234722788, "mean": 0.000157460202672155, "normalized": [ 0.7615886502877881, 0.869859991005712, 0.7823534284439785, 0.735201115807512, 0.9596786286759813, 0.8813992955944876, 0.8128285893567467, 0.7239431731448671, 0.5977864740547744, 0.5606030792342883, 0.5318673348078621, 0.5130567490014816, 0.6987134321316841, 0.6407269063222457, 0.6075335461477536, 0.5704889146486948, 0.7180428901779269, 0.6784140170824181, 0.6819219612414157, 0.7908928021790416, 3.2176296651190497, 2.545498211708922, 2.0288816729118655, 1.6601468184271124, 2.321862751456626, 1.7738386620365922, 1.1711907122839438, 0.9284857615000071, 0.7896970255450443, 0.5236030848875767, 0.47341239614817127, 0.4488522586284289 ], "peak_layer": 21, "peak_normalized": 3.2176296651190497, "population_cv": 0.6625973254291921, "rest_mean": 0.0001179550758765631, "spike_contrast": 3.143466990402101, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00037078788736835124, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011991990322712809, 0.00013696833048015833, 0.00012318952940404415, 0.00011576491669984534, 0.0001511111913714558, 0.00013878531171940267, 0.00012798815441783518, 0.00011399223876651376, 9.412757935933769e-05, 8.82726744748652e-05, 8.374793833354488e-05, 8.078601968009025e-05, 0.000110019558633212, 0.00010088898852700368, 9.566235530655831e-05, 8.982930012280121e-05, 0.0001130631790147163, 0.00010682320862542838, 0.00010737557022366673, 0.00012453414092306048, 0.0005066486191935837, 0.00040081466431729496, 0.00031946811941452324, 0.0002614070544950664, 0.0003656009794212878, 0.00027930899523198605, 0.00018441592692397535, 0.00014619955618400127, 0.00012434585369192064, 8.24466478661634e-05, 7.454361184500158e-05, 7.067636761348695e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0311521975836513e-05, "first_to_last_ratio": 1.1418129560476864, "imbalance_abs_log_ratio": 0.13261731152138012, "last_quartile_mean": 9.030832870848826e-06, "mean": 1.3945096057454975e-05, "normalized": [ 0.7145337044030123, 0.932335477800909, 0.645487324949067, 0.558162738870377, 0.34661223918607953, 0.7511338800577496, 0.9447212820009926, 1.0225104899919193, 0.6315142693979814, 0.5614582240786927, 0.4574638139025647, 0.40615350248375953, 0.4194348381092698, 0.7040945119904685, 0.6483274463519548, 0.5709286433652233, 0.8217341836062412, 1.1893729541204203, 1.2408864574521172, 1.5700693567578727, 2.5502861886942485, 4.114342551839228, 2.785136777749787, 2.232505646034574, 1.1806687356339707, 1.2053474712098855, 0.7898562388784587, 0.5945521897391881, 0.46830824879649285, 0.2867747481020556, 0.32588927285831853, 0.32939659158711965 ], "peak_layer": 22, "peak_normalized": 4.114342551839228, "population_cv": 0.8262244130468728, "rest_mean": 9.884005235276331e-06, "spike_contrast": 3.629600110811422, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.587498649721965e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.964241144189145e-06, 1.3001507795706857e-05, 9.001382750284392e-06, 7.783633009239566e-06, 4.833540970139438e-06, 1.0474634109414183e-05, 1.3174229025025852e-05, 1.425900700269267e-05, 8.80652714840835e-06, 7.82958886702545e-06, 6.3793768276809715e-06, 5.663849606207805e-06, 5.849059107276844e-06, 9.818665603233967e-06, 9.040988516062498e-06, 7.961654773680493e-06, 1.1459162124083377e-05, 1.658592009334825e-05, 1.730428084556479e-05, 2.189476799685508e-05, 3.556398587534204e-05, 5.7374902098672464e-05, 3.883899989887141e-05, 3.113250568276271e-05, 1.6464538930449635e-05, 1.68086862686323e-05, 1.1014621122740209e-05, 8.291087397083174e-06, 6.530603513965616e-06, 3.999101409135619e-06, 4.544557214103406e-06, 4.593467110680649e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.944479362871789e-05, "first_to_last_ratio": 1.1135282861542686, "imbalance_abs_log_ratio": 0.1075336103151947, "last_quartile_mean": 3.54232524841791e-05, "mean": 5.533300526394669e-05, "normalized": [ 0.5280207224829799, 0.8999098868607942, 0.6284748654319019, 0.5778568458898549, 0.34988006202074273, 0.7764713506345448, 0.9190909808587969, 1.023190827540606, 0.6226417920568781, 0.5519145939388094, 0.4568960180345979, 0.4213932325876586, 0.4074494455716704, 0.7465319443901276, 0.6474109861274324, 0.5764712279753077, 0.7979703549327781, 1.2369697451324837, 1.2324489817728508, 1.5808985864286207, 2.5860289117498976, 4.232714993086349, 2.835646318943954, 2.2422528664695807, 1.1646676435019565, 1.2175941057769888, 0.783401544111113, 0.5289102464485725, 0.46735876466607296, 0.2966943922188728, 0.3266155865457417, 0.3362221758114643 ], "peak_layer": 22, "peak_normalized": 4.232714993086349, "population_cv": 0.8496980089222349, "rest_mean": 3.881239232820614e-05, "spike_contrast": 3.7241794809927478, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014454431511694564, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.921697341662366e-05, 4.9794718506745994e-05, 3.477540303720161e-05, 3.197455589543097e-05, 1.9359915313543752e-05, 4.296449333196506e-05, 5.085606608190574e-05, 5.661622344632633e-05, 3.4452641557436436e-05, 3.053909313166514e-05, 2.5281429770984687e-05, 2.3316953956964426e-05, 2.2545402316609398e-05, 4.1307856008643284e-05, 3.582319550332613e-05, 3.189788549207151e-05, 4.415409784996882e-05, 6.844525341875851e-05, 6.819510599598289e-05, 8.747586980462074e-05, 0.0001430927513865754, 0.00023420884099323303, 0.00015690483269281685, 0.00012407058966346085, 6.444456084864214e-05, 6.737314106430858e-05, 4.3347961764084175e-05, 2.9266193450894207e-05, 2.5860364985419437e-05, 1.641699236643035e-05, 1.807262196962256e-05, 1.8604183424031362e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03498003748245537, "first_to_last_ratio": 1.1135283344223057, "imbalance_abs_log_ratio": 0.10753365366212776, "last_quartile_mean": 0.03141369321383536, "mean": 0.04906986153218895, "normalized": [ 0.528020735774383, 0.8999099163028887, 0.6284749107820657, 0.5778568640801447, 0.34988006993508275, 0.776471348923419, 0.919091002023713, 1.0231908018037408, 0.622641782085556, 0.5519145889381298, 0.45689603549337854, 0.4213932430966296, 0.4074494429214576, 0.7465319861987265, 0.6474110467766695, 0.5764712070514831, 0.7979703658517325, 1.236969773835328, 1.2324490791670337, 1.5808986244806464, 2.5860289675304746, 4.232714870782151, 2.835646138092529, 2.242252864087746, 1.1646675839359508, 1.2175940341247224, 0.7834015320190452, 0.528910268066999, 0.467358763362216, 0.2966943974186558, 0.32661557776013733, 0.33622217729716364 ], "peak_layer": 22, "peak_normalized": 4.232714870782151, "population_cv": 0.8496979833458974, "rest_mean": 0.034419217595347655, "spike_contrast": 3.72417933196884, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12818333879113197, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.0259099043905735, 0.0441584549844265, 0.03083917684853077, 0.02835535630583763, 0.017168566584587097, 0.03810134157538414, 0.04509966820478439, 0.05020783096551895, 0.030552946031093597, 0.027082372456789017, 0.022419825196266174, 0.020677708089351654, 0.019993487745523453, 0.036632221192121506, 0.03176837041974068, 0.028287362307310104, 0.039156295359134674, 0.06069793552160263, 0.06047610566020012, 0.0775744765996933, 0.12689608335494995, 0.2076987326145172, 0.13914476335048676, 0.11002703756093979, 0.05715007707476616, 0.05974717065691948, 0.038441404700279236, 0.02595355361700058, 0.022933229804039, 0.014558752998709679, 0.01602698117494583, 0.016498375684022903 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.750052800593039e-06, "first_to_last_ratio": 0.4308304317433478, "imbalance_abs_log_ratio": 0.8420406961220296, "last_quartile_mean": 6.383144267374519e-06, "mean": 4.5530042669383874e-06, "normalized": [ 2.0777537806616246, 0.8353410859824805, 0.1910970826520623, 0.2941663520688202, 0.18051642612757607, 0.3343816911052641, 0.3564217354179324, 0.562389007550559, 0.34578362377189115, 0.35956396375636174, 0.3051237753595574, 0.33780495264365074, 0.3038063276744268, 0.5221079484446093, 0.49376666786286805, 0.5929821493588614, 0.3686987288124583, 0.7392110518834059, 0.5965749796632107, 0.9039444986896679, 1.6537371809131796, 3.0803086468030596, 2.8630898340929525, 2.485723197420275, 1.724168620325731, 2.41211172564755, 1.9299601312238395, 1.6334426636441552, 1.3339260437132254, 0.9558102182238872, 0.6013349904117006, 0.6249509180931564 ], "peak_layer": 22, "peak_normalized": 3.0803086468030596, "population_cv": 0.8333981127885269, "rest_mean": 3.4051366684353625e-06, "spike_contrast": 3.1574325337710922, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0751489298854722e-05, "top_five_layers": [ 22, 23, 24, 26, 1 ], "top_five_spike_overlap": 3 }, "values": [ 9.460021828999743e-06, 3.80331152882718e-06, 8.700658327143174e-07, 1.3393406561590382e-06, 8.218920584113221e-07, 1.5224412663883413e-06, 1.622789682187431e-06, 2.56055955105694e-06, 1.5743543144708383e-06, 1.6370962612199946e-06, 1.389229851156415e-06, 1.5380273907794617e-06, 1.383231506224547e-06, 2.3771597170707537e-06, 2.248121745651588e-06, 2.699850256249192e-06, 1.678686885497882e-06, 3.3656310733931605e-06, 2.71620842795528e-06, 4.1156631596095394e-06, 7.529472441092366e-06, 1.4024658412381541e-05, 1.3035660231253132e-05, 1.1317508324282244e-05, 7.850147085264325e-06, 1.0982354979205411e-05, 8.787116712483112e-06, 7.437071417371044e-06, 6.073370968806557e-06, 4.35180800195667e-06, 2.737880777203827e-06, 2.8454041967052035e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6044932951663213e-05, "first_to_last_ratio": 1.1246393261284713, "imbalance_abs_log_ratio": 0.11746238525662386, "last_quartile_mean": 2.315847609679622e-05, "mean": 3.690820258839267e-05, "normalized": [ 0.45689793146127605, 0.8738188272728166, 0.6345310201461108, 0.5870234156151962, 0.3959041381169584, 0.8145991894129532, 0.9605505534893847, 0.9220179377165436, 0.6534489876582346, 0.574567881594872, 0.47330901138441, 0.4295574998000546, 0.4505301728221963, 0.8093943395842734, 0.6650197233282829, 0.5723463492360998, 0.7892367771238881, 1.2091363576952083, 1.2417470885367035, 1.5367390457984618, 2.460873737770247, 4.227774989574034, 3.004456821926061, 2.2368262162922274, 1.1669129504749511, 1.3012136399895533, 0.7651380597325796, 0.47754123850698527, 0.4891606669175266, 0.2974921422063077, 0.24654864385331124, 0.27568464496229084 ], "peak_layer": 22, "peak_normalized": 4.227774989574034, "population_cv": 0.8549059646786937, "rest_mean": 2.5840054991864168e-05, "spike_contrast": 3.7413310319999478, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.667619960964657e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.6863281416590326e-05, 3.2251082302536815e-05, 2.341939944017213e-05, 2.166597914765589e-05, 1.4612110135203693e-05, 3.006539191119373e-05, 3.545219442457892e-05, 3.4030024835374206e-05, 2.4117627617670223e-05, 2.120626777468715e-05, 1.7468984879087657e-05, 1.5854195225983858e-05, 1.6628258890705183e-05, 2.9873290259274654e-05, 2.4544682673877105e-05, 2.1124275008332916e-05, 2.9129310860298574e-05, 4.462704964680597e-05, 4.583065310725942e-05, 5.671827602782287e-05, 9.082642645807937e-05, 0.00015603957581333816, 0.00011088910105172545, 8.255723514594138e-05, 4.3068659579148516e-05, 4.8025456635514274e-05, 2.823987051669974e-05, 1.7625188775127754e-05, 1.805404099286534e-05, 1.0979900253005326e-05, 9.099667295231484e-06, 1.0175024726777337e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.700510824775847e-05, "first_to_last_ratio": 1.1234226056354075, "imbalance_abs_log_ratio": 0.1163799234305708, "last_quartile_mean": 1.5136875617827172e-05, "mean": 2.416565462226572e-05, "normalized": [ 0.4144047676027832, 0.875113510868426, 0.648392503192907, 0.5978987553197436, 0.4472461776170949, 0.8581546509904864, 0.9915956817879598, 0.7967069167230741, 0.6805807808463835, 0.6196150111411964, 0.5091007553108428, 0.44255818216184833, 0.49857092482915655, 0.8652430602969526, 0.6897944860164179, 0.5800177420993363, 0.7901724317041942, 1.1871172558602836, 1.243251110659502, 1.5081298165112624, 2.2807123105787017, 4.096637658310102, 3.1477300512572848, 2.2202178168572866, 1.1479124533540228, 1.3987972507734237, 0.7581679674914413, 0.4356681137302007, 0.511763265149204, 0.3118761594955322, 0.21315449183792956, 0.2336979396250189 ], "peak_layer": 22, "peak_normalized": 4.096637658310102, "population_cv": 0.8395518179977153, "rest_mean": 1.710104003938829e-05, "spike_contrast": 3.6439054716132255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.231457336980384e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.001436248770915e-05, 2.114769085892476e-05, 1.5668829291826114e-05, 1.4448614820139483e-05, 1.0807996659423225e-05, 2.0737868908327073e-05, 2.396255877101794e-05, 1.9252944184700027e-05, 1.644668009248562e-05, 1.497340235800948e-05, 1.2302753020776436e-05, 1.0694708180380985e-05, 1.2048292774125002e-05, 2.090916495944839e-05, 1.6669335309416056e-05, 1.4016508430358954e-05, 1.9095034076599404e-05, 2.868746560125146e-05, 3.0043976948945783e-05, 3.644494427135214e-05, 5.511490599019453e-05, 9.899793076328933e-05, 7.606695726281032e-05, 5.365301694837399e-05, 2.7740055884351023e-05, 3.3802851248765364e-05, 1.8321625248063356e-05, 1.0528205166338012e-05, 1.236709431395866e-05, 7.536691555287689e-06, 5.1510178309399635e-06, 5.647463694913313e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.642002219654387e-05, "first_to_last_ratio": 1.1203608181269622, "imbalance_abs_log_ratio": 0.11365079246685211, "last_quartile_mean": 6.821018814662239e-05, "mean": 0.00011010971388714097, "normalized": [ 0.5163109854936148, 0.8673817094155928, 0.5924673517780332, 0.5654643004921971, 0.34399113467095715, 0.7609201839652954, 0.9076117604661024, 0.9981345295043799, 0.6327009050241252, 0.5203303165972624, 0.4437849422865529, 0.41423566265400885, 0.39884159677379066, 0.7686876572920235, 0.6339609024838357, 0.5703063832410542, 0.781268072457197, 1.1797447140196398, 1.2760717444836593, 1.5599326259342914, 2.6126342040159805, 4.419910912775975, 2.9287458133972417, 2.3507635425478965, 1.1736586910474214, 1.2201215929679614, 0.7838637940961916, 0.4869646019277815, 0.46617791476107356, 0.26920984270675297, 0.25672408594104645, 0.2990775247810641 ], "peak_layer": 22, "peak_normalized": 4.419910912775975, "population_cv": 0.8912016880930539, "rest_mean": 7.550380838370916e-05, "spike_contrast": 3.9333327677514873, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002969816036056727, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.685085488948971e-05, 9.550715185469016e-05, 6.523641059175134e-05, 6.226311234058812e-05, 3.787676541833207e-05, 8.378470374736935e-05, 9.993687126552686e-05, 0.00010990430746460333, 6.966651562834159e-05, 5.729342228733003e-05, 4.886503302259371e-05, 4.561137029668316e-05, 4.3916334107052535e-05, 8.463997801300138e-05, 6.980525358812883e-05, 6.279627268668264e-05, 8.602520392742008e-05, 0.00012990135292056948, 0.0001405078946845606, 0.00017176373512484133, 0.0002876764046959579, 0.0004866751260124147, 0.0003224833635613322, 0.0002588419010862708, 0.00012923122267238796, 0.0001343472395092249, 8.631101809442043e-05, 5.3619532991433516e-05, 5.13307168148458e-05, 2.9642618756042793e-05, 2.8267815650906414e-05, 3.293134068371728e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.6955231027823174e-06, "first_to_last_ratio": 0.14308275980953641, "imbalance_abs_log_ratio": 1.9443320762007668, "last_quartile_mean": 3.2816833481774665e-05, "mean": 1.7864957968072304e-05, "normalized": [ 0.5367890577532618, 0.2021050510263675, 0.1411976282823933, 0.2660218188543664, 0.19890690834720978, 0.16151559172232732, 0.2167464101659875, 0.3793916565295996, 0.424775345287454, 0.3093373886676644, 0.2783613720705816, 0.2825637413001494, 0.3733172205433837, 0.3886765425479794, 0.44060863402002287, 0.44323494933287805, 0.3667341230574315, 0.45728447370296305, 0.42556973592368313, 0.5658685710706646, 2.6227066493164064, 3.0085127149589037, 2.58733070947387, 2.226933740479959, 2.8743975419651484, 2.84666799381699, 2.39504866979417, 1.5908588860103314, 1.8185955715005, 1.4035945808786974, 0.9127606447015644, 0.8535860768970909 ], "peak_layer": 22, "peak_normalized": 3.0085127149589037, "population_cv": 0.9632431009960319, "rest_mean": 1.2359982755933069e-05, "spike_contrast": 3.850476578600324, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.759182411362417e-05, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 9.589713954483159e-06, 3.6105982417211635e-06, 2.5224896944564534e-06, 4.7524686124233995e-06, 3.5534635571821127e-06, 2.8854692573077045e-06, 3.872165507345926e-06, 6.777815997338621e-06, 7.588593689433765e-06, 5.526299446501071e-06, 4.972914211975876e-06, 5.047989361628424e-06, 6.669296453765128e-06, 6.9436900957953185e-06, 7.871454727137461e-06, 7.918373739812523e-06, 6.5516896938788705e-06, 8.1693679021555e-06, 7.6027854447602294e-06, 1.0109218237630557e-05, 4.685454405262135e-05, 5.3746953199151903e-05, 4.622255437425338e-05, 3.97840776713565e-05, 5.135099127073772e-05, 5.085560405859724e-05, 4.278744381736033e-05, 2.8420627131708898e-05, 3.248913344577886e-05, 2.507515819161199e-05, 1.6306430552504025e-05, 1.5249279385898262e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.8878105947806034e-05, "first_to_last_ratio": 0.1437388462831867, "imbalance_abs_log_ratio": 1.939757193388504, "last_quartile_mean": 0.00013133614493199275, "mean": 7.144442986373178e-05, "normalized": [ 0.543944422065902, 0.20498362677674017, 0.14117605251679072, 0.2675823635170904, 0.19429426405696296, 0.15941273479469137, 0.21634544427316113, 0.386139638870723, 0.42865458135412043, 0.3129364237141443, 0.2788876084512782, 0.2815826978190512, 0.3736543391719335, 0.3909795972451435, 0.4389936626557141, 0.44232523153890874, 0.3688366584716598, 0.45947150441297413, 0.42734480687548726, 0.5800824758005463, 2.6187902433353907, 2.987231504916195, 2.580721746390286, 2.2092452217225382, 2.879951595578818, 2.8406783220158585, 2.397028447242086, 1.5906421267626043, 1.8247593500794728, 1.4018234858691145, 0.9198949147470885, 0.8516049069575247 ], "peak_layer": 22, "peak_normalized": 2.987231504916195, "population_cv": 0.9601858610516129, "rest_mean": 4.954554700547036e-05, "spike_contrast": 3.828767846228416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018969839729834347, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.886179911205545e-05, 1.4644938346464187e-05, 1.008624258247437e-05, 1.911726940306835e-05, 1.3881242921343073e-05, 1.1389151950425003e-05, 1.5456676919711754e-05, 2.7587526346906088e-05, 3.062498217332177e-05, 2.2357564375852235e-05, 1.9924966181861237e-05, 2.0117515305173583e-05, 2.6695521228248253e-05, 2.7933314413530752e-05, 3.13636519422289e-05, 3.160167398164049e-05, 2.6351324777351692e-05, 3.282667967141606e-05, 3.0531406082445756e-05, 4.144366175751202e-05, 0.00018709797586780041, 0.00021342105173971504, 0.0001843781938077882, 0.00015783826529514045, 0.0002057564997812733, 0.0002029506431426853, 0.00017125433078035712, 0.00011364251986378804, 0.00013036889140494168, 0.00010015247971750796, 6.572136771865189e-05, 6.08424270467367e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.016741293016821146, "first_to_last_ratio": 0.14373884545887533, "imbalance_abs_log_ratio": 1.9397571991232891, "last_quartile_mean": 0.11647020652890205, "mean": 0.06335763493552804, "normalized": [ 0.5439444372437541, 0.20498362168576487, 0.14117604516877305, 0.2675823445899417, 0.1942942529879486, 0.15941273039509152, 0.2163454315588509, 0.38613966530889043, 0.4286545697273512, 0.3129364189494029, 0.278887605667477, 0.2815826915705289, 0.3736543321474427, 0.3909795777927573, 0.43899363576159356, 0.44232520797796643, 0.3688366746353626, 0.45947150248290297, 0.4273447896704886, 0.580082454376481, 2.6187903327724635, 2.987231533325295, 2.5807218070831297, 2.2092452282911403, 2.8799516329199686, 2.84067822247012, 2.397028505583422, 1.5906421518499794, 1.8247592808395534, 1.4018234450909468, 0.9198949415397243, 0.8516049285354858 ], "peak_layer": 22, "peak_normalized": 2.987231533325295, "population_cv": 0.9601858709388058, "rest_mean": 0.04393748607900408, "spike_contrast": 3.8287679558695995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.16822643876075744, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.03446303308010101, 0.01298727747052908, 0.00894458033144474, 0.01695338450372219, 0.012310024350881577, 0.010100013576447964, 0.013707134872674942, 0.024464895948767662, 0.02715853974223137, 0.01982691138982773, 0.017669659107923508, 0.017840413376688957, 0.023673854768276215, 0.024771541357040405, 0.02781359851360321, 0.02802467904984951, 0.02336861938238144, 0.029111027717590332, 0.02707555517554283, 0.03675265237689018, 0.16592036187648773, 0.18926392495632172, 0.16350843012332916, 0.13997255265712738, 0.18246692419052124, 0.1799786537885666, 0.15187005698680878, 0.10077932476997375, 0.11561243236064911, 0.08881621807813644, 0.05828236788511276, 0.053955674171447754 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0825735472508313e-06, "first_to_last_ratio": 0.12187576812955331, "imbalance_abs_log_ratio": 2.104753047082524, "last_quartile_mean": 8.882598763193528e-06, "mean": 4.656443639561303e-06, "normalized": [ 0.5462983207096154, 0.1799290732737773, 0.13161762512632208, 0.2712494331106157, 0.1613911681124062, 0.13237661267085615, 0.15721616286614848, 0.2798363906909718, 0.34662434040104295, 0.2682122377700395, 0.24445710198520895, 0.24681111820039023, 0.3364691861798734, 0.3619231630019147, 0.42744951901080175, 0.43112099033231693, 0.36249737820633765, 0.4368387762566429, 0.42578881412518377, 0.38711372693052853, 2.644134376023103, 3.0274176414789977, 2.680305221816438, 2.2521785297578925, 2.965193895729998, 2.901136489590974, 2.603841896063781, 1.6862657755052308, 1.872923926339135, 1.3894795988927409, 0.9615945171774488, 0.8803069926632656 ], "peak_layer": 22, "peak_normalized": 3.0274176414789977, "population_cv": 1.0082602587575777, "rest_mean": 3.1785867888713795e-06, "spike_contrast": 3.9756254815914156, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2636870633286889e-05, "top_five_layers": [ 22, 25, 26, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 2.543807340771309e-06, 8.3782958881784e-07, 6.128700533736264e-07, 1.2630576975425356e-06, 7.515088782383827e-07, 6.164042360978783e-07, 7.32068201614311e-07, 1.3030423815507675e-06, 1.6140367051775684e-06, 1.2489151686168043e-06, 1.138300717684615e-06, 1.14926206151722e-06, 1.5667498018956394e-06, 1.6852748103701742e-06, 1.990394594031386e-06, 2.0074905933142873e-06, 1.687948611106549e-06, 2.034115141213988e-06, 1.9826616153295618e-06, 1.8025732515525306e-06, 1.2312262697378173e-05, 1.409699962096056e-05, 1.2480690202210099e-05, 1.0487142390047666e-05, 1.3807258255837951e-05, 1.3508978554455098e-05, 1.2124643035349436e-05, 7.85200154496124e-06, 8.721164704184048e-06, 6.470033440564293e-06, 4.477610673347954e-06, 4.099099896848202e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2557812397062662e-05, "first_to_last_ratio": 0.15959566659574662, "imbalance_abs_log_ratio": 1.8351117459907644, "last_quartile_mean": 7.868517150200205e-05, "mean": 4.5438396199415365e-05, "normalized": [ 0.5876869810338485, 0.2186593308079107, 0.115659623677568, 0.24728992953724943, 0.2138823134903866, 0.17309343907534852, 0.22828222046505, 0.4264066618299725, 0.47941738273017553, 0.34947398487973863, 0.31642185290041636, 0.3326608895247323, 0.42753132054545495, 0.43329608717407087, 0.4807098955780001, 0.491094994316893, 0.41296023678331367, 0.5280992844682807, 0.4827771475203192, 0.6138123378227869, 2.7117823469683264, 3.061642080099734, 2.646250140140893, 2.167597408443836, 2.9446942269972274, 2.694165709502317, 2.239316799454017, 1.5139318638986488, 1.6066586018929343, 1.2634426578175135, 0.8290147862338136, 0.7622874643892231 ], "peak_layer": 22, "peak_normalized": 3.061642080099734, "population_cv": 0.9412918554701797, "rest_mean": 3.107991987808082e-05, "spike_contrast": 3.9567080229621103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012297416833462194, "top_five_layers": [ 22, 25, 21, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.6703553885454312e-05, 9.935529305948876e-06, 5.255387804936618e-06, 1.1236457794439048e-05, 9.718469300423749e-06, 7.86508826422505e-06, 1.0372777978773229e-05, 1.9375234842300415e-05, 2.178395698138047e-05, 1.587953738635406e-05, 1.4377701518242247e-05, 1.511557729827473e-05, 1.9426337530603632e-05, 1.968827928067185e-05, 2.1842686692252755e-05, 2.231456892332062e-05, 1.876425085356459e-05, 2.3995984520297498e-05, 2.1936619305051863e-05, 2.789064819808118e-05, 0.00012321904068812728, 0.0001391161058563739, 0.00012024136231048033, 9.849214984569699e-05, 0.00013380218297243118, 0.00012241856893524528, 0.00010175096394959837, 6.879063585074618e-05, 7.300399011000991e-05, 5.740880806115456e-05, 3.766910231206566e-05, 3.463711982476525e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.367274347165221e-06, "first_to_last_ratio": 0.1920014554991945, "imbalance_abs_log_ratio": 1.6502523262581177, "last_quartile_mean": 4.3579223529377487e-05, "mean": 2.787530080183842e-05, "normalized": [ 0.6522901960458961, 0.23732362933486428, 0.10383521258256681, 0.24073339013803968, 0.24132146388193534, 0.19483084771733855, 0.2475207738927628, 0.483488789493259, 0.550030749638162, 0.4005903921146771, 0.3626794403805512, 0.398664404925254, 0.5057551655640742, 0.49633936232785464, 0.5444163811542189, 0.5645330449101175, 0.481087325236049, 0.6303807295780811, 0.5534160889368265, 0.6675682370643486, 2.937265214456223, 3.231070824381384, 2.7157464748293116, 2.0522050942602807, 2.8791759008165254, 2.406039643209466, 1.9404796965168836, 1.3618287054639213, 1.3825321378732942, 1.123237294980051, 0.7355691904186271, 0.6780441978771551 ], "peak_layer": 22, "peak_normalized": 3.231070824381384, "population_cv": 0.9137826743997257, "rest_mean": 1.8774052764288002e-05, "spike_contrast": 4.1025792976954865, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.702204020461067e-05, "top_five_layers": [ 22, 21, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.8182785424869508e-05, 6.615467555093346e-06, 2.8944377845618874e-06, 6.710515663144179e-06, 6.726908395648934e-06, 5.430968485597987e-06, 6.899716026964597e-06, 1.3477395441441331e-05, 1.5332272596424446e-05, 1.1166577678523026e-05, 1.0109798495250288e-05, 1.111289020627737e-05, 1.409807737218216e-05, 1.3835609024681617e-05, 1.5175770386122167e-05, 1.5736528439447284e-05, 1.3410453902906738e-05, 1.7572052456671372e-05, 1.5426639947691e-05, 1.86086654139217e-05, 8.187715138774365e-05, 9.006707114167511e-05, 7.570224988739938e-05, 5.720583430957049e-05, 8.025789429666474e-05, 6.706907879561186e-05, 5.409145524026826e-05, 3.796138480538502e-05, 3.8538499211426824e-05, 3.131057746941224e-05, 2.0504212443483993e-05, 1.890068597276695e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.742885974133969e-05, "first_to_last_ratio": 0.13285814939480156, "imbalance_abs_log_ratio": 2.018473265828943, "last_quartile_mean": 0.00028172046586405486, "mean": 0.00014909372896454443, "normalized": [ 0.5351570886371335, 0.1998816464903299, 0.1134003859562838, 0.2373819769757024, 0.18971959520637977, 0.1492636791238371, 0.21004413698536548, 0.3734913450927867, 0.42927632587153325, 0.3024206473119055, 0.27462175768495817, 0.27240928064339276, 0.3512855646324889, 0.3655094888356237, 0.41973324501001746, 0.42402062842094873, 0.360798486332806, 0.4559662563090299, 0.41985007515873884, 0.5597866310716989, 2.4844541141001115, 2.96085466615825, 2.5443412288142064, 2.249909733543605, 2.9984626521607822, 2.895026684700318, 2.601768175835175, 1.7136369023458127, 1.8313353225818492, 1.363588302533558, 0.8854239763588411, 0.82717999911653 ], "peak_layer": 25, "peak_normalized": 2.9984626521607822, "population_cv": 0.9800748961570518, "rest_mean": 0.00010360345199673989, "spike_contrast": 3.810116525877056, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003947412245906889, "top_five_layers": [ 25, 22, 26, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 7.978856592671946e-05, 2.980110002681613e-05, 1.6907286408240907e-05, 3.53921641362831e-05, 2.8286001906963065e-05, 2.2254278519540094e-05, 3.131626363028772e-05, 5.568521737586707e-05, 6.400240818038583e-05, 4.508902202360332e-05, 4.0944381908047944e-05, 4.061451545567252e-05, 5.2374474762473255e-05, 5.449517266242765e-05, 6.257959466893226e-05, 6.321881664916873e-05, 5.379279173212126e-05, 6.79817094351165e-05, 6.259701331146061e-05, 8.34606762509793e-05, 0.0003704165283124894, 0.00044144486309960485, 0.0003793453215621412, 0.0003354474320076406, 0.0004470519779715687, 0.0004316303238738328, 0.00038790731923654675, 0.00025549251586198807, 0.00027304061222821474, 0.00020330246479716152, 0.00013201116234995425, 0.00012332735059317201 ] } } } } }, "uniform_value_backward": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_value_backward", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.459922923975682e-06, "first_to_last_ratio": 2.328407895763024, "imbalance_abs_log_ratio": 0.8451847274778195, "last_quartile_mean": 3.6333509001451603e-06, "mean": 5.48445950698806e-06, "normalized": [ 1.7981492239015215, 1.9357513407040896, 1.6672073870519302, 1.5017840953582675, 1.4824821685817133, 1.4014170610824435, 1.3280238040790173, 1.2253951633993214, 1.156695262052324, 1.0641431906294738, 1.0106344255365511, 0.9662588157958233, 0.9475559543612537, 0.9090700068173221, 0.8837794995113782, 0.8574858790556309, 0.8607721566700557, 0.837358195636748, 0.8360590737297285, 0.824566471544106, 0.8217232949937042, 0.8060327579708028, 0.7938760845984737, 0.7839296928623085, 0.7744755711517991, 0.7571739836659549, 0.7307992717355416, 0.68650330244616, 0.6478013597166409, 0.6078235943688355, 0.5594006660114121, 0.5358712449796666 ], "peak_layer": 2, "peak_normalized": 1.9357513407040896, "population_cv": 0.35649081874965816, "rest_mean": 5.691642602161866e-06, "spike_contrast": 0.7670317864637676, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.365670793049503e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 9.8618766060099e-06, 1.0616549843689427e-05, 9.14373140403768e-06, 8.236474059231114e-06, 8.130613423418254e-06, 7.686015123908874e-06, 7.283492777787615e-06, 6.720630153722595e-06, 6.343848326650914e-06, 5.836250238644425e-06, 5.542783583223354e-06, 5.299407348502427e-06, 5.196832262299722e-06, 4.985757641406963e-06, 4.847052878176328e-06, 4.7028465814946685e-06, 4.720870037999703e-06, 4.59245711681433e-06, 4.585332135320641e-06, 4.522301424003672e-06, 4.506708137341775e-06, 4.420654022396775e-06, 4.353981239546556e-06, 4.2994306568289176e-06, 4.247579909133492e-06, 4.152690053160768e-06, 4.008039013569942e-06, 3.7650995636795415e-06, 3.552840325937723e-06, 3.333583890707814e-06, 3.068010300921742e-06, 2.93896414405026e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.357288642291678e-05, "first_to_last_ratio": 2.3058080996835404, "imbalance_abs_log_ratio": 0.8354312005538636, "last_quartile_mean": 1.456013899314712e-05, "mean": 2.1838724165945678e-05, "normalized": [ 1.8092622254038118, 1.9431844817635626, 1.6767107875669782, 1.4950940749576347, 1.4674045069810522, 1.3807616632021802, 1.3135886403618173, 1.2124726583027057, 1.1458806799401573, 1.058561922179424, 1.0064538267556071, 0.9652676350426349, 0.9479497443118112, 0.9127224235896364, 0.8880734273823512, 0.8621668896719061, 0.8678176638273486, 0.8415578850574983, 0.8383782990632195, 0.8250966517885899, 0.8217974585881554, 0.8072470254003167, 0.7943189523527976, 0.7845346497712516, 0.7759095209279144, 0.7590478200773931, 0.7335564093275885, 0.6907997066984136, 0.653253330518448, 0.6137700422385202, 0.5655418513252468, 0.5418171456240263 ], "peak_layer": 2, "peak_normalized": 1.9431844817635626, "population_cv": 0.3545327058404385, "rest_mean": 2.266066288332998e-05, "spike_contrast": 0.7678616985591674, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.740025509207044e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.951197868445888e-05, 4.2436669900780544e-05, 3.661722439574078e-05, 3.2650947105139494e-05, 3.204624226782471e-05, 3.01540731015848e-05, 2.8687099984381348e-05, 2.6478855943423696e-05, 2.502457209629938e-05, 2.3117641831049696e-05, 2.197966750827618e-05, 2.1080213628010824e-05, 2.0702012989204377e-05, 1.99326932488475e-05, 1.9394390619709156e-05, 1.8828624888556078e-05, 1.895203058666084e-05, 1.8378550521447323e-05, 1.8309112419956364e-05, 1.8019058188656345e-05, 1.7947008018381894e-05, 1.762924512149766e-05, 1.7346912500215694e-05, 1.7133235814981163e-05, 1.694487400527578e-05, 1.6576635971432552e-05, 1.6019936083466746e-05, 1.5086184248502832e-05, 1.4266219295677729e-05, 1.3403954653767869e-05, 1.2350712495390326e-05, 1.1832595191663131e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.029772771056741476, "first_to_last_ratio": 2.3058080657449813, "imbalance_abs_log_ratio": 0.8354311858351369, "last_quartile_mean": 0.012912076897919178, "mean": 0.01936679895152338, "normalized": [ 1.8092622688469415, 1.9431844502992837, 1.6767106129751548, 1.4950940451422423, 1.4674045233760984, 1.3807616273635772, 1.3135886581915137, 1.2124727235166237, 1.1458806648362485, 1.0585619314023544, 1.0064538755889925, 0.965267610260004, 0.9479497452013365, 0.91272242891937, 0.8880734542201515, 0.8621668660127682, 0.8678176632029413, 0.8415579121933264, 0.838378389023671, 0.8250966015781978, 0.8217974339297908, 0.8072470687715084, 0.7943189250144355, 0.7845346707619233, 0.7759095445302433, 0.7590478000025082, 0.7335564570017248, 0.69079971362327, 0.6532533237931528, 0.6137700206530842, 0.56554185062535, 0.5418171391422113 ], "peak_layer": 2, "peak_normalized": 1.9431844502992837, "population_cv": 0.35453269447871943, "rest_mean": 0.020095702398706367, "spike_contrast": 0.7678617064775293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01543072033673525, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.035039618611335754, 0.0376332625746727, 0.03247251734137535, 0.028955185785889626, 0.028418928384780884, 0.02674093283712864, 0.02544000744819641, 0.023481715470552444, 0.02219204045832157, 0.02050095610320568, 0.019491789862513542, 0.018694143742322922, 0.018358752131462097, 0.01767651177942753, 0.017199140042066574, 0.016697412356734276, 0.01680685020983219, 0.016298282891511917, 0.01623670570552349, 0.015979479998350143, 0.015915585681796074, 0.01563379168510437, 0.015383414924144745, 0.015193925239145756, 0.015026884153485298, 0.014700326137244701, 0.014206640422344208, 0.013378579169511795, 0.012651425786316395, 0.011886760592460632, 0.01095273531973362, 0.010493263602256775 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2918531382410947e-06, "first_to_last_ratio": 1.348602552566167, "imbalance_abs_log_ratio": 0.2990689100691611, "last_quartile_mean": 9.57919837674126e-07, "mean": 1.0475390865849477e-06, "normalized": [ 2.4104888609370185, 1.341971980631633, 1.2193546382838214, 1.0858339683324785, 0.9788660798673142, 0.9472203180914999, 0.9376150883665126, 0.9444624163032238, 0.9288836138476254, 0.8900745462090585, 0.8740769359854389, 0.8711050178927995, 0.8545471342557541, 0.8634403690695975, 0.8786271700589888, 0.8875789012156576, 0.8758221673193468, 0.9032444171738859, 0.9303783636244995, 0.9552629683285443, 0.9757615411235937, 1.0157402642998257, 1.045251506463968, 1.0688091449644375, 1.0624071055149489, 1.0529656438443655, 1.0284038000374691, 0.9523781978486857, 0.8838469105126692, 0.8403571044364565, 0.7713914356307573, 0.7238323895281237 ], "peak_layer": 1, "peak_normalized": 2.4104888609370185, "population_cv": 0.279812062407651, "rest_mean": 1.0410222465132527e-06, "spike_contrast": 1.040064250882766, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0827300229721004e-06, "top_five_layers": [ 1, 2, 3, 4, 24 ], "top_five_spike_overlap": 1 }, "values": [ 2.5250812996091554e-06, 1.4057681028134539e-06, 1.2773216440109536e-06, 1.1374535233699135e-06, 1.0254004791931948e-06, 9.922503068082733e-07, 9.821884532357217e-07, 9.893612968880916e-07, 9.730418923936668e-07, 9.323878771283489e-07, 9.156297551271564e-07, 9.125165547629877e-07, 8.951715244620573e-07, 9.044875355357362e-07, 9.203963031723106e-07, 9.297735914515215e-07, 9.174579531645577e-07, 9.461838317292859e-07, 9.746077012096066e-07, 1.0006752972913091e-06, 1.0221483535133302e-06, 1.0640276286721928e-06, 1.0949418083328055e-06, 1.1196193554496858e-06, 1.1129129688924877e-06, 1.103022668758058e-06, 1.0772931773317396e-06, 9.976533874578308e-07, 9.258641853193694e-07, 8.803069135865371e-07, 8.08062679880095e-07, 7.582427201668906e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.57942979260406e-05, "first_to_last_ratio": 2.5994532681374096, "imbalance_abs_log_ratio": 0.9553011414295731, "last_quartile_mean": 9.922970434672607e-06, "mean": 1.6124295910913133e-05, "normalized": [ 1.8750767487966784, 2.0012252521436076, 1.745806554118046, 1.5722176381119681, 1.5471673963385955, 1.4484453745095827, 1.357829461828116, 1.2499614439371438, 1.1713452937825866, 1.079951327288068, 1.0230197185281116, 0.9769120487640461, 0.96163457802513, 0.918624119720732, 0.8885326600597652, 0.8580725357821459, 0.8699071437806652, 0.8325299891167686, 0.8310410601256768, 0.8095951692808093, 0.8030390759939747, 0.7755468823771603, 0.7515517005264444, 0.7277277649695101, 0.7158336494479539, 0.6950266607011315, 0.6717468100420292, 0.6353600749013578, 0.6043990157307626, 0.5679033620811016, 0.527236654105826, 0.5057328350845046 ], "peak_layer": 2, "peak_normalized": 2.0012252521436076, "population_cv": 0.3945192515369112, "rest_mean": 1.6856638096753698e-05, "spike_contrast": 0.7219498951999067, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2169648107374087e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.0234292353270575e-05, 3.226834814995527e-05, 2.8149901481810957e-05, 2.5350902433274314e-05, 2.4946984922280535e-05, 2.3355161829385906e-05, 2.189404403907247e-05, 2.0154748199274763e-05, 1.8887118130805902e-05, 1.741345477057621e-05, 1.6495472664246336e-05, 1.575201895320788e-05, 1.550568049424328e-05, 1.4812167137279175e-05, 1.4326963537314441e-05, 1.3835815479978919e-05, 1.4026640201336704e-05, 1.3423959899228066e-05, 1.3399951967585366e-05, 1.305415207752958e-05, 1.2948439689353108e-05, 1.2505147424235474e-05, 1.2118242011638358e-05, 1.1734097824955825e-05, 1.154231358668767e-05, 1.1206815543118864e-05, 1.0831444342329632e-05, 1.0244733857689425e-05, 9.745508577907458e-06, 9.157041858998127e-06, 8.501319825882092e-06, 8.154585884767585e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.933050248226209e-05, "first_to_last_ratio": 2.5350541410379415, "imbalance_abs_log_ratio": 0.930214994226497, "last_quartile_mean": 7.6252819098954205e-06, "mean": 1.202020879986776e-05, "normalized": [ 1.8583274986098608, 2.0026775386155595, 1.77731729208076, 1.615557162502337, 1.5930029936690748, 1.4693486681854961, 1.3292468480215616, 1.2198575705059131, 1.1444407348951777, 1.053479376420273, 1.0059382961834429, 0.9516573870963836, 0.9421936613647472, 0.8964225087904136, 0.8652818603489327, 0.841617098226079, 0.8649772378810286, 0.8234169272301419, 0.8325562065776818, 0.8056805753420667, 0.8017546832979361, 0.7661992033221119, 0.7431607861543773, 0.720913207047878, 0.7139696155781592, 0.6926020821557612, 0.6726446950161074, 0.6435870710508805, 0.6274403778164145, 0.5970363543218487, 0.569837411659171, 0.5578570700324221 ], "peak_layer": 2, "peak_normalized": 2.0026775386155595, "population_cv": 0.3945939876253568, "rest_mean": 1.2578481834713818e-05, "spike_contrast": 0.7159476421745723, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.005534411699046e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.2337484551826492e-05, 2.4072602172964253e-05, 2.136372495442629e-05, 1.941933442139998e-05, 1.91482286027167e-05, 1.7661877791397274e-05, 1.5977824659785256e-05, 1.4662942703580484e-05, 1.3756416592514142e-05, 1.2663042070926167e-05, 1.20915883599082e-05, 1.143912049883511e-05, 1.1325364539516158e-05, 1.0775185728562064e-05, 1.0400868632132187e-05, 1.0116413250216283e-05, 1.039720700646285e-05, 9.897643394651823e-06, 1.0007499440689571e-05, 9.68444874160923e-06, 9.63725869951304e-06, 9.209874406224117e-06, 8.932947821449488e-06, 8.66552727529779e-06, 8.582063856010791e-06, 8.325221642735414e-06, 8.08532968221698e-06, 7.736050974926911e-06, 7.5419643508212175e-06, 7.176501640060451e-06, 6.849564670119435e-06, 6.705558462272165e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.031241830190993e-05, "first_to_last_ratio": 2.261102148101721, "imbalance_abs_log_ratio": 0.8158523705600311, "last_quartile_mean": 3.109652448074485e-05, "mean": 4.619880877498872e-05, "normalized": [ 1.7997064857450336, 1.904359030499371, 1.6374224610397496, 1.477941957137803, 1.4602990373876177, 1.3729930118356377, 1.3118711870701552, 1.2110312951649602, 1.137824927128826, 1.0604309170276205, 1.0061171703075658, 0.9633053240172016, 0.949330006688662, 0.9216248238136175, 0.906779203920096, 0.8687912686710114, 0.8800028320072268, 0.8475838252231725, 0.8463171144866274, 0.8286039531613457, 0.8201877997739442, 0.8102510580287974, 0.7986912791711968, 0.7937156242787844, 0.7857004463402506, 0.7744091131435279, 0.7453649305566711, 0.6981461341444608, 0.6620680295240863, 0.614979715499695, 0.5681574453782887, 0.5359925918269958 ], "peak_layer": 2, "peak_normalized": 1.904359030499371, "population_cv": 0.3447198298829091, "rest_mean": 4.7895252632938706e-05, "spike_contrast": 0.7733128004546086, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.70380119420588e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.314429578604177e-05, 8.797911868896335e-05, 7.564696716144681e-05, 6.827915785834193e-05, 6.746407598257065e-05, 6.343064160319045e-05, 6.060688610887155e-05, 5.594820322585292e-05, 5.256615622784011e-05, 4.899064515484497e-05, 4.648141475627199e-05, 4.4503558456199244e-05, 4.3857915443368256e-05, 4.2577968997647986e-05, 4.189211904304102e-05, 4.013712168671191e-05, 4.06550825573504e-05, 3.915736306225881e-05, 3.909884253516793e-05, 3.828051558230072e-05, 3.789169932133518e-05, 3.74326336896047e-05, 3.689858567668125e-05, 3.666871634777635e-05, 3.629842467489652e-05, 3.577677853172645e-05, 3.443497189437039e-05, 3.225351974833757e-05, 3.058675429201685e-05, 2.8411330276867375e-05, 2.624819717311766e-05, 2.476221925462596e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.247537380157155e-06, "first_to_last_ratio": 2.3850511827649155, "imbalance_abs_log_ratio": 0.8692205843888265, "last_quartile_mean": 3.458012741930361e-06, "mean": 5.339330805043119e-06, "normalized": [ 1.6764909218312523, 1.9617954794601216, 1.6992151272822646, 1.4983517248278486, 1.5139649706461762, 1.4256520166778042, 1.3385091192829222, 1.2434304822395104, 1.131080096564001, 1.0546107928438107, 1.0037428856625374, 0.962940037127743, 0.9549245789646706, 0.9150321056456306, 0.8876946170122294, 0.8650767880832936, 0.8770248559584294, 0.8519848957954361, 0.8533470944296186, 0.848236507394223, 0.8353817265855179, 0.8173190953401269, 0.8073880080861434, 0.7956133450140085, 0.7777311024585761, 0.7505722139587175, 0.705927759165694, 0.665757931848141, 0.624138138877991, 0.5752620873586926, 0.5511417007015933, 0.530661792875274 ], "peak_layer": 2, "peak_normalized": 1.9617954794601216, "population_cv": 0.3568485733648799, "rest_mean": 5.530472267141535e-06, "spike_contrast": 0.7788063481128092, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.307166909711668e-06, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.95133962330874e-06, 1.0474675036675762e-05, 9.07267167349346e-06, 8.000195521162823e-06, 8.08355980552733e-06, 7.612027729919646e-06, 7.146742973418441e-06, 6.639086677751038e-06, 6.039210802555317e-06, 5.630915893561905e-06, 5.359315309760859e-06, 5.141455403645523e-06, 5.098658220958896e-06, 4.885659109277185e-06, 4.7396952140843496e-06, 4.6189311433408875e-06, 4.682725830207346e-06, 4.549029199552024e-06, 4.556302428682102e-06, 4.52901531389216e-06, 4.460379386728164e-06, 4.363937023299513e-06, 4.310911663196748e-06, 4.2480428419366945e-06, 4.1525636333972216e-06, 4.007553343399195e-06, 3.7691818306484492e-06, 3.5547018342185766e-06, 3.3324799915135372e-06, 3.071514584007673e-06, 2.942727860499872e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.269888588874892e-05, "first_to_last_ratio": 2.35686858292009, "imbalance_abs_log_ratio": 0.8573338663198418, "last_quartile_mean": 1.3873868965674774e-05, "mean": 2.1262430664137355e-05, "normalized": [ 1.6815888767055769, 1.9686225336479115, 1.7075392779747713, 1.4904980805963362, 1.4972655559630919, 1.4034192556792997, 1.3238794766841318, 1.2301588485530683, 1.121552358208752, 1.0499507133536148, 1.0011277236394078, 0.963324273966619, 0.9560838812000779, 0.9203885412533225, 0.8930508699850467, 0.8703824880945366, 0.8841528709344532, 0.8558102501916335, 0.8550558750774316, 0.8483637687825304, 0.8350347355926667, 0.8178886567183555, 0.8078366810386547, 0.7969743808996977, 0.7797462600958652, 0.7532932557652127, 0.7097594040214169, 0.6713244282550651, 0.6299481338236045, 0.5812721180769863, 0.5573951790036309, 0.5373112462172303 ], "peak_layer": 2, "peak_normalized": 1.9686225336479115, "population_cv": 0.35389080715057486, "rest_mean": 2.202041212984585e-05, "spike_contrast": 0.7797006998811187, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7169330749311485e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.5754666896536946e-05, 4.1857700125547126e-05, 3.630643550422974e-05, 3.169161209370941e-05, 3.183550506946631e-05, 2.9840104616596363e-05, 2.8148895580670796e-05, 2.615616722323466e-05, 2.3846929252613336e-05, 2.232450424344279e-05, 2.1286408809828572e-05, 2.0482615582295693e-05, 2.0328667233115993e-05, 1.9569697542465292e-05, 1.89884322026046e-05, 1.850644730438944e-05, 1.8799239114741795e-05, 1.819660610635765e-05, 1.818056625779718e-05, 1.8038275811704807e-05, 1.7754868167685345e-05, 1.7390300854458474e-05, 1.717657141853124e-05, 1.6945612514973618e-05, 1.6579300790908746e-05, 1.601684562047012e-05, 1.5091210116224829e-05, 1.4273989108914975e-05, 1.339422851742711e-05, 1.2359258107608184e-05, 1.1851576346089132e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02899769856594503, "first_to_last_ratio": 2.35686858681256, "imbalance_abs_log_ratio": 0.8573338679713849, "last_quartile_mean": 0.012303485535085201, "mean": 0.01885573595063761, "normalized": [ 1.6815888871597915, 1.9686224147635019, 1.7075392481901774, 1.4904981407970117, 1.4972654379699921, 1.4034192484903574, 1.3238795515861768, 1.230158916579243, 1.1215523220126975, 1.0499507103306343, 1.001127702787151, 0.9633242487753337, 0.9560838749935686, 0.9203885704718483, 0.8930508868493195, 0.8703824268783725, 0.8841529174326813, 0.8558103041085099, 0.8550558906482283, 0.8483637681044575, 0.8350347432767428, 0.8178887044245051, 0.8078366908478906, 0.796974401455123, 0.7797462730931964, 0.7532932002519721, 0.7097594341790385, 0.671324454011456, 0.6299481306480355, 0.5812721137067387, 0.5573951795880433, 0.5373112055882039 ], "peak_layer": 2, "peak_normalized": 1.9686224147635019, "population_cv": 0.35389079505518783, "rest_mean": 0.019527921418624895, "spike_contrast": 0.7797007217053022, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01522593442350626, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.03170759603381157, 0.037119824439287186, 0.03219690918922424, 0.02810443937778473, 0.028232041746377945, 0.026462502777576447, 0.02496272325515747, 0.023195551708340645, 0.021147694438695908, 0.019797593355178833, 0.018876999616622925, 0.01816418766975403, 0.018027665093541145, 0.017354603856801987, 0.016839131712913513, 0.01641170121729374, 0.016671353951096535, 0.016136933118104935, 0.016122708097100258, 0.015996523201465607, 0.015745194628834724, 0.015421893447637558, 0.015232355333864689, 0.015027538873255253, 0.014702689833939075, 0.014203897677361965, 0.013383036479353905, 0.012658316642045975, 0.011878135614097118, 0.010960313491523266, 0.010510096326470375, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2314776824950968e-06, "first_to_last_ratio": 1.358248310832468, "imbalance_abs_log_ratio": 0.30619586281350625, "last_quartile_mean": 9.066660879852861e-07, "mean": 1.0095481766114744e-06, "normalized": [ 2.2510647132267514, 1.3455279053230764, 1.2283491143156429, 1.0526455105583856, 1.0056690176197902, 0.9692221599384264, 0.9572558258596283, 0.9489099547933215, 0.9218216280115407, 0.8905546788106925, 0.8763660110671972, 0.8694242377636043, 0.8685998082234971, 0.8764693322122661, 0.8891057616302812, 0.8990022944596364, 0.9003146700723137, 0.9275413402074646, 0.9581989480285121, 0.9876176030015524, 1.000156116631407, 1.0336678655116023, 1.068099651182219, 1.0896881962272793, 1.081885225695443, 1.0551976273002857, 0.9988041017347361, 0.9238330401452145, 0.8601167762256441, 0.7958830056879117, 0.7583180274077601, 0.7106898511269164 ], "peak_layer": 1, "peak_normalized": 2.2510647132267514, "population_cv": 0.2561818803841662, "rest_mean": 9.993219375953938e-07, "spike_contrast": 1.0654923375948289, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0647698672983098e-06, "top_five_layers": [ 1, 2, 3, 24, 25 ], "top_five_spike_overlap": 2 }, "values": [ 2.2725582766724983e-06, 1.3583752433987684e-06, 1.2400776085996768e-06, 1.0626963558024727e-06, 1.015271323012712e-06, 9.784764642972732e-07, 9.663958735472988e-07, 9.579703146300744e-07, 9.306233437200717e-07, 8.990578521661519e-07, 8.847337085171603e-07, 8.777256539360678e-07, 8.768933525971079e-07, 8.848380161907698e-07, 8.975951004686067e-07, 9.075861271412577e-07, 9.089110335480655e-07, 9.363976687382092e-07, 9.673480008132174e-07, 9.970475502996123e-07, 1.0097057838720502e-06, 1.0435375088491128e-06, 1.078298055290361e-06, 1.1000927315762965e-06, 1.092215256903728e-06, 1.0652728406057577e-06, 1.0083408596983645e-06, 9.326539611720364e-07, 8.683293231115385e-07, 8.03482237188291e-07, 7.655585818611144e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.5231804784198175e-05, "first_to_last_ratio": 2.6518592370690897, "imbalance_abs_log_ratio": 0.975260992886681, "last_quartile_mean": 9.514760222373297e-06, "mean": 1.575830160049918e-05, "normalized": [ 1.7680072391258728, 2.020274701367778, 1.7747981337484644, 1.5702283057720683, 1.5745659549387958, 1.468575316323847, 1.366589194311488, 1.2663643116143115, 1.1465258163389551, 1.0704343447928568, 1.0164163108220898, 0.9740484512911409, 0.9677541966052103, 0.9245201462352831, 0.8913652028193111, 0.8644158057124367, 0.8839251267896815, 0.845522595189969, 0.8460497665068152, 0.8329662909412521, 0.8145821291394524, 0.7836795820242509, 0.7613757450733065, 0.7366672589275867, 0.716422356677449, 0.689594392508022, 0.652884419046894, 0.6205919149770609, 0.5818976326649927, 0.5425153265661802, 0.5218602418605108, 0.5045817892866661 ], "peak_layer": 2, "peak_normalized": 2.020274701367778, "population_cv": 0.39415073781885135, "rest_mean": 1.645124252212958e-05, "spike_contrast": 0.7304263253994938, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2016420623695012e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.786079130601138e-05, 3.1836098060011864e-05, 2.7967804271611385e-05, 2.47441312239971e-05, 2.4812485207803547e-05, 2.314225275767967e-05, 2.1535124687943608e-05, 1.9955750758526847e-05, 1.8067299606627785e-05, 1.6868227248778567e-05, 1.6016994777601212e-05, 1.5349349268944934e-05, 1.5250162505253684e-05, 1.4568867300113197e-05, 1.4046401702216826e-05, 1.362172497465508e-05, 1.3929158740211278e-05, 1.3324000065040309e-05, 1.3332307389646303e-05, 1.3126134035701398e-05, 1.2836430869356263e-05, 1.2349459211691283e-05, 1.1997988622169942e-05, 1.1608624845393933e-05, 1.128959956986364e-05, 1.0866836419154424e-05, 1.0288349585607648e-05, 9.77947456703987e-06, 9.169718396151438e-06, 8.549120138923172e-06, 8.223631084547378e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.902213432458666e-05, "first_to_last_ratio": 2.570514437449803, "imbalance_abs_log_ratio": 0.9441060490896753, "last_quartile_mean": 7.4001274015245144e-06, "mean": 1.1811577877551827e-05, "normalized": [ 1.7836661639182594, 2.007884486098661, 1.801766001704916, 1.6166376372836335, 1.6162792780207194, 1.484195322536222, 1.334217273990145, 1.2390748916935237, 1.1152682343795186, 1.047183670438877, 0.9989511624599826, 0.9468241347561922, 0.942586809823197, 0.9015408911709663, 0.8679632288380377, 0.8434512704540663, 0.8735354304753469, 0.8326703084602718, 0.8456967523674954, 0.8281893546686905, 0.8104136880244638, 0.7723543788975116, 0.750047939286133, 0.727483933767052, 0.7080168021618385, 0.6848377954753971, 0.6576536196741141, 0.6404010158732426, 0.6084114841371344, 0.5799389397954393, 0.5702129585195616, 0.5626451408493897 ], "peak_layer": 2, "peak_normalized": 2.007884486098661, "population_cv": 0.39402258867818274, "rest_mean": 1.2350397237578701e-05, "spike_contrast": 0.720782753960385, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.901953333406708e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.1067911802674644e-05, 2.3716283976682462e-05, 2.1281699446262792e-05, 1.909504135255702e-05, 1.9090808564214967e-05, 1.753068863763474e-05, 1.57592112373095e-05, 1.4635429579357151e-05, 1.3173077604733407e-05, 1.2368891475489363e-05, 1.1799189451267011e-05, 1.1183487004018389e-05, 1.1133437510579824e-05, 1.0648620445863344e-05, 1.025201527227182e-05, 9.962490366888233e-06, 1.0317831765860319e-05, 9.835150194703601e-06, 9.989013051381335e-06, 9.782223060028628e-06, 9.572264389134943e-06, 9.12272389541613e-06, 8.859249646775424e-06, 8.592733138357289e-06, 8.36279559734976e-06, 8.089014954748563e-06, 7.767926945234649e-06, 7.564146471850108e-06, 7.186299626482651e-06, 6.849993951618671e-06, 6.735114766343031e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.828012237747316e-05, "first_to_last_ratio": 2.312494882445189, "imbalance_abs_log_ratio": 0.8383269774053195, "last_quartile_mean": 2.9526604748753016e-05, "mean": 4.489596511803029e-05, "normalized": [ 1.6577187409379064, 1.9152258678535772, 1.6744752052558183, 1.4840054420576252, 1.4912917773323804, 1.3961062438002583, 1.3297764454700043, 1.2182169964281564, 1.1079961846053805, 1.047881406618081, 0.999963947398217, 0.9691026873070004, 0.9556277478959141, 0.9353258396988691, 0.9030474230504885, 0.8874013307551517, 0.8935396144488046, 0.8678070689766392, 0.8641197392760416, 0.8577613745480667, 0.8343135878548937, 0.8204103169462279, 0.8193971014627366, 0.8081501178832716, 0.7960663239792694, 0.7673062057216204, 0.7200712732146042, 0.6825993169002439, 0.6333797250658595, 0.5852510577399719, 0.5516412156589899, 0.5250226738579302 ], "peak_layer": 2, "peak_normalized": 1.9152258678535772, "population_cv": 0.3428609763636811, "rest_mean": 4.642851807282703e-05, "spike_contrast": 0.7887432268392901, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6620179162127894e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.442488276865333e-05, 8.598591375630349e-05, 7.517718040617183e-05, 6.662585656158626e-05, 6.695298361591995e-05, 6.267953722272068e-05, 5.970159691059962e-05, 5.469302777783014e-05, 4.9744558054953814e-05, 4.704564707935788e-05, 4.489434650167823e-05, 4.350880044512451e-05, 4.2903830035356805e-05, 4.199235627311282e-05, 4.054318560520187e-05, 3.984073919127695e-05, 4.0116323361871764e-05, 3.89610358979553e-05, 3.879548967233859e-05, 3.851002475130372e-05, 3.745731373783201e-05, 3.683311297209002e-05, 3.6787623685086146e-05, 3.628267950261943e-05, 3.574016591301188e-05, 3.4448952646926045e-05, 3.232829476473853e-05, 3.064595512114465e-05, 2.8436194043024443e-05, 2.627541107358411e-05, 2.4766464775893837e-05, 2.357139965170063e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.4598292460214e-06, "first_to_last_ratio": 2.328358854359362, "imbalance_abs_log_ratio": 0.8451636650513356, "last_quartile_mean": 3.633387194668103e-06, "mean": 5.48445886039417e-06, "normalized": [ 1.7981292044822659, 1.9357505739332324, 1.6671812164387254, 1.501771337574118, 1.4824621119466825, 1.4013991506965175, 1.3280020709214548, 1.225379388067507, 1.156718780628782, 1.0641415748591472, 1.0106456553801202, 0.9662668896138349, 0.9475625334929744, 0.9090604128642483, 0.8837694050828951, 0.8574965104342549, 0.860785690483099, 0.8373501686260958, 0.8360756725074443, 0.8245861368451315, 0.8217235577025693, 0.8060327700829376, 0.7938754319523651, 0.7839311948498927, 0.7744962255348613, 0.7571785503773947, 0.7308065715532428, 0.6865019323584283, 0.6478069914366877, 0.6078375958541098, 0.5594073652125656, 0.5358673282064139 ], "peak_layer": 2, "peak_normalized": 1.9357505739332324, "population_cv": 0.3564831577662627, "rest_mean": 5.691637507307287e-06, "spike_contrast": 0.7670365797994658, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.365694167063339e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 9.861765647656284e-06, 1.0616544386721216e-05, 9.143586794380099e-06, 8.236403118644375e-06, 8.130502465064637e-06, 7.68591598898638e-06, 7.283372724486981e-06, 6.720542842231225e-06, 6.343976565403864e-06, 5.836240688950056e-06, 5.542844519368373e-06, 5.2994510042481124e-06, 5.196867732593091e-06, 4.985704435966909e-06, 4.846996944252169e-06, 4.702904334408231e-06, 4.720943707070546e-06, 4.592412551573943e-06, 4.5854226300434675e-06, 4.522408744378481e-06, 4.506709046836477e-06, 4.420653567649424e-06, 4.353977146820398e-06, 4.299438387533883e-06, 4.247692686476512e-06, 4.152714609517716e-06, 4.008078576589469e-06, 3.765091605600901e-06, 3.552870794010232e-06, 3.333660288262763e-06, 3.068046680709813e-06, 2.9389423161774175e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.3572764550626744e-05, "first_to_last_ratio": 2.3057895392595262, "imbalance_abs_log_ratio": 0.8354231510987864, "last_quartile_mean": 1.456020333989727e-05, "mean": 2.183873047556517e-05, "normalized": [ 1.80924687671537, 1.9431940819531333, 1.676696643261768, 1.4950864798932781, 1.467401917430866, 1.3807559335924544, 1.31358592866812, 1.212462979303528, 1.1458997558869217, 1.0585578682053984, 1.0064671958441547, 0.9652690219967264, 0.9479597986271983, 0.9127134975291503, 0.888075836142126, 0.862166224116184, 0.867826991667279, 0.841547063844033, 0.83837247628253, 0.8251061585552277, 0.8217963049445071, 0.8072479582581175, 0.7943165572696724, 0.7845285926712973, 0.7759202912841717, 0.7590490167365564, 0.7335626108658329, 0.6907982993808188, 0.6532520589865599, 0.6137777359935852, 0.565539938799635, 0.5418179052937989 ], "peak_layer": 2, "peak_normalized": 1.9431940819531333, "population_cv": 0.3545305270645445, "rest_mean": 2.2660667733968392e-05, "spike_contrast": 0.76786216030628, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7400269280187784e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.951165490434505e-05, 4.2436891817487776e-05, 3.6616926081478596e-05, 3.265079067205079e-05, 3.204619497410022e-05, 3.0153956686262973e-05, 2.868704905267805e-05, 2.64786522166105e-05, 2.5024995920830406e-05, 2.3117559976526536e-05, 2.197996582253836e-05, 2.1080250007798895e-05, 2.0702238543890417e-05, 1.993250407394953e-05, 1.939444882737007e-05, 1.882861579360906e-05, 1.8952239770442247e-05, 1.8378319509793073e-05, 1.8308990547666326e-05, 1.801927101041656e-05, 1.7946988009498455e-05, 1.762927058734931e-05, 1.7346865206491202e-05, 1.7133108485722914e-05, 1.6945114111877047e-05, 1.6576666894252412e-05, 1.602007614565082e-05, 1.508615787315648e-05, 1.4266195648815483e-05, 1.3404126548266504e-05, 1.2350674296612851e-05, 1.183261520054657e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.029772662557661533, "first_to_last_ratio": 2.3057895177959966, "imbalance_abs_log_ratio": 0.8354231417902483, "last_quartile_mean": 0.01291213370859623, "mean": 0.019366804277524352, "normalized": [ 1.809246767643357, 1.9431941106945074, 1.6766966870594806, 1.495086420690814, 1.4674019077542524, 1.3807559578984756, 1.3135858925156472, 1.2124629647121858, 1.1458997775072552, 1.0585579855568523, 1.0064672559708643, 0.9652689798174655, 0.9479597754699817, 0.9127135219659304, 0.888075806778596, 0.8621662442022726, 0.8678270422679951, 0.841547005089253, 0.8383724840087661, 0.8251060885690371, 0.8217963423351232, 0.8072480008996415, 0.7943165425842438, 0.7845285881999872, 0.7759202953515476, 0.7590489858287814, 0.7335626029644629, 0.6907983214336281, 0.653252038106227, 0.6137777383932114, 0.5655399639078199, 0.5418179038223359 ], "peak_layer": 2, "peak_normalized": 1.9431941106945074, "population_cv": 0.35453052191257556, "rest_mean": 0.020095706399944093, "spike_contrast": 0.7678621746036595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.015430732816457748, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.035039328038692474, 0.03763346001505852, 0.03247225657105446, 0.028955046087503433, 0.02841888554394245, 0.02674083039164543, 0.02543996088206768, 0.02348153293132782, 0.022192416712641716, 0.02050088532269001, 0.019492054358124733, 0.01869417540729046, 0.018358951434493065, 0.017676344141364098, 0.017199190333485603, 0.01669740490615368, 0.016807036474347115, 0.016298076137900352, 0.016236595809459686, 0.015979668125510216, 0.01591556891798973, 0.01563381403684616, 0.015383373014628887, 0.015193811617791653, 0.01502709649503231, 0.014700353145599365, 0.014206763356924057, 0.01337855588644743, 0.012651404365897179, 0.01188691332936287, 0.010952701792120934, 0.010493281297385693 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.291835857841761e-06, "first_to_last_ratio": 1.3485833026747331, "imbalance_abs_log_ratio": 0.299054636012817, "last_quartile_mean": 9.579206974308363e-07, "mean": 1.0475393885656104e-06, "normalized": [ 2.4104197937214296, 1.3419993768147092, 1.2193873876623726, 1.0857681046985204, 0.9788139215354608, 0.9471793472155275, 0.9376610507921448, 0.9444495548464527, 0.9288857336777386, 0.8900930106166185, 0.8741329012581984, 0.8711139916120108, 0.8545363607420399, 0.8634605233315061, 0.8786425447326217, 0.887541854523117, 0.8758151318715187, 0.9032250016851439, 0.9304164056283383, 0.9552919953757107, 0.9757865467437721, 1.0156631340115454, 1.0452801819870623, 1.0688491005570937, 1.0624669234868362, 1.0529715263673087, 1.0284586355464043, 0.9523438456637114, 0.8838175160851754, 0.8403596296329269, 0.7713843760240694, 0.7237845915529142 ], "peak_layer": 1, "peak_normalized": 2.4104197937214296, "population_cv": 0.27980440532718537, "rest_mean": 1.0410195853991254e-06, "spike_contrast": 1.0400825698677958, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0827463256646297e-06, "top_five_layers": [ 1, 2, 3, 4, 24 ], "top_five_spike_overlap": 1 }, "values": [ 2.5250096769013908e-06, 1.4057972066439106e-06, 1.2773563184964587e-06, 1.1373848565199296e-06, 1.0253461368847638e-06, 9.922076742441277e-07, 9.82236883828591e-07, 9.893481092149159e-07, 9.730443935040967e-07, 9.324074881078559e-07, 9.156886449090962e-07, 9.12526218144194e-07, 8.951604968387983e-07, 9.045089086612279e-07, 9.204126740769425e-07, 9.297350516135339e-07, 9.174508477372001e-07, 9.46163766002428e-07, 9.746478326633223e-07, 1.000705992737494e-06, 1.0221748425465194e-06, 1.063947138391086e-06, 1.0949721627184772e-06, 1.1196615332664805e-06, 1.1129759514005855e-06, 1.1030291489078081e-06, 1.0773509302453022e-06, 9.976176897907862e-07, 9.258336604034412e-07, 8.80309812600899e-07, 8.080555176093185e-07, 7.581928684885497e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.5794245857468923e-05, "first_to_last_ratio": 2.5994382822780953, "imbalance_abs_log_ratio": 0.9552953764086378, "last_quartile_mean": 9.923007610268542e-06, "mean": 1.6124308444886992e-05, "normalized": [ 1.875069425095799, 2.001233172595225, 1.7457889523480976, 1.5722148366283213, 1.547165404000032, 1.4484413155137394, 1.3578243451672811, 1.2499566367465564, 1.1713539721385289, 1.0799457497690719, 1.023025015060715, 0.976913996826235, 0.9616437578274982, 0.9186228415910193, 0.888533041071715, 0.8580748018430988, 0.8699143078938142, 0.8325216144516643, 0.8310392860253453, 0.8096019290349791, 0.8030372108501562, 0.7755478024583314, 0.7515516803716358, 0.7277252251002508, 0.7158404256809536, 0.6950255563813353, 0.6717498413974154, 0.6353591861780038, 0.6043991099627453, 0.567907771476934, 0.5272362442668266, 0.5057355442466751 ], "peak_layer": 2, "peak_normalized": 2.001233172595225, "population_cv": 0.3945173779757133, "rest_mean": 1.685664924648504e-05, "spike_contrast": 0.7219506046728209, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2169668116257526e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.023419776582159e-05, 3.226850094506517e-05, 2.8149639547336847e-05, 2.5350876967422664e-05, 2.494697218935471e-05, 2.3355114535661414e-05, 2.189397855545394e-05, 2.0154686353635043e-05, 1.8887272744905204e-05, 1.741337837302126e-05, 1.6495570889674127e-05, 1.5752062608953565e-05, 1.550584056531079e-05, 1.4812158042332157e-05, 1.4326980817713775e-05, 1.3835862773703411e-05, 1.402676662110025e-05, 1.3423835298453923e-05, 1.339993377769133e-05, 1.3054271221335512e-05, 1.294841968046967e-05, 1.2505171980592422e-05, 1.2118251106585376e-05, 1.1734065992641263e-05, 1.15424318209989e-05, 1.1206806448171847e-05, 1.0831501640495844e-05, 1.0244727491226513e-05, 9.745517672854476e-06, 9.15712007554248e-06, 8.501319825882092e-06, 8.154635906976182e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9329483848196105e-05, "first_to_last_ratio": 2.5350525368598045, "imbalance_abs_log_ratio": 0.9302143614279234, "last_quartile_mean": 7.624884915458097e-06, "mean": 1.201984051135696e-05, "normalized": [ 1.8583357088751307, 2.0027277021690506, 1.7771668453163478, 1.615570343516219, 1.592938001514604, 1.4694493793664982, 1.32898385240296, 1.2198799651106356, 1.144267264787803, 1.0533998962206053, 1.0058282278662807, 0.9519161926020662, 0.9421831838426558, 0.8966453451440455, 0.8650037408265971, 0.8416374374551236, 0.8651561323934935, 0.8233100437191515, 0.8327440956360556, 0.8058278405503257, 0.8017291581294494, 0.7663098470747268, 0.7431513228704318, 0.7209725235957172, 0.7140172181236788, 0.6925010270670932, 0.6728195124367922, 0.6435648715599169, 0.6274519981756614, 0.5969246531502274, 0.5697514359189264, 0.5578352325817294 ], "peak_layer": 2, "peak_normalized": 2.0027277021690506, "population_cv": 0.3945781723821696, "rest_mean": 1.2578015162876841e-05, "spike_contrast": 0.7159871630405812, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.005697393149603e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.233689883723855e-05, 2.407246756774839e-05, 2.1361262042773888e-05, 1.941889786394313e-05, 1.9146860722685233e-05, 1.766254717949778e-05, 1.597417394805234e-05, 1.4662762623629533e-05, 1.3753910025116056e-05, 1.2661698747251648e-05, 1.20898948807735e-05, 1.144188081525499e-05, 1.1324891602271236e-05, 1.0777534043882042e-05, 1.039720700646285e-05, 1.0116347766597755e-05, 1.039903872879222e-05, 9.896055416902527e-06, 1.0009451216319576e-05, 9.685922123026103e-06, 9.636656614020467e-06, 9.210922144120559e-06, 8.932560376706533e-06, 8.665974746691063e-06, 8.582373084209394e-06, 8.32375189929735e-06, 8.087183232419193e-06, 7.735547114862129e-06, 7.541872946603689e-06, 7.174939128162805e-06, 6.848321390862111e-06, 6.705090527248103e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.030938422758481e-05, "first_to_last_ratio": 2.2606879822899777, "imbalance_abs_log_ratio": 0.8156691839012263, "last_quartile_mean": 3.110087936875061e-05, "mean": 4.619929336513451e-05, "normalized": [ 1.8000040071532064, 1.904408036353138, 1.637904058859854, 1.4777606171460849, 1.4597770726472121, 1.372671031064488, 1.3115103960008867, 1.2109361461416157, 1.137534863820107, 1.0602352149716379, 1.0062919047911225, 0.9633358523801769, 0.9491225557491253, 0.9216727196154594, 0.9066025950003278, 0.8688487743640587, 0.8802774784785187, 0.8475087887298653, 0.846208545767534, 0.8286626678374982, 0.8203744064059802, 0.810217596933452, 0.7988918917164441, 0.7937267489942904, 0.7857342550389422, 0.7744580018919649, 0.7454551502724039, 0.6984329250267984, 0.662100300186566, 0.6150609478307189, 0.5682149152709306, 0.5360595335595907 ], "peak_layer": 2, "peak_normalized": 1.904408036353138, "population_cv": 0.3446893198253748, "rest_mean": 4.789507282246767e-05, "spike_contrast": 0.7734007302345921, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.704208429553546e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.315891318488866e-05, 8.798230555839837e-05, 7.567001011921093e-05, 6.82714962749742e-05, 6.744066922692582e-05, 6.341643165796995e-05, 6.0590853536268696e-05, 5.5944394262041897e-05, 5.255330688669346e-05, 4.8982117732521147e-05, 4.6489974920405075e-05, 4.45054356532637e-05, 4.384879139252007e-05, 4.258062836015597e-05, 4.188439925201237e-05, 4.014019941678271e-05, 4.066819747094996e-05, 3.915430716006085e-05, 3.9094236853998154e-05, 3.828362969215959e-05, 3.790071787079796e-05, 3.743148045032285e-05, 3.690824087243527e-05, 3.6669614928541705e-05, 3.630036735557951e-05, 3.5779412428382784e-05, 3.443950117798522e-05, 3.2267107599182054e-05, 3.058856600546278e-05, 2.841538116626907e-05, 2.6251127565046772e-05, 2.4765571652096696e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.247530786320567e-06, "first_to_last_ratio": 2.3850533729557593, "imbalance_abs_log_ratio": 0.86922150268771, "last_quartile_mean": 3.45800680179309e-06, "mean": 5.339328467357518e-06, "normalized": [ 1.676519761737885, 1.9618007671893674, 1.6991821441606432, 1.498358513038152, 1.5139533691039981, 1.4256686527077282, 1.3385028917629673, 1.243419273267211, 1.1310636430692016, 1.054615257538969, 1.0037504793539562, 0.9629245320498924, 0.9549124771530781, 0.9150161537455905, 0.8877036077745802, 0.8650752931079952, 0.8770263471429458, 0.8519858650005171, 0.8533548777823373, 0.8482425851215958, 0.8353975931646603, 0.8173266925801457, 0.8073981560598078, 0.7956149708934007, 0.7777226705206242, 0.7505720315608271, 0.7059216805334544, 0.665764185190204, 0.6241330890536816, 0.5752674068009126, 0.5511430066219813, 0.5306620252116886 ], "peak_layer": 2, "peak_normalized": 1.9618007671893674, "population_cv": 0.3568479486512281, "rest_mean": 5.530464544857817e-06, "spike_contrast": 0.7788122704557785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.3071936488559e-06, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.951489689934533e-06, 1.0474698683538008e-05, 9.072491593542509e-06, 8.000228262972087e-06, 8.083494321908802e-06, 7.612113222421613e-06, 7.14670659363037e-06, 6.639023922616616e-06, 6.039120307832491e-06, 5.630937266687397e-06, 5.359353508538334e-06, 5.141370365890907e-06, 5.098591373098316e-06, 4.885571797785815e-06, 4.739741143566789e-06, 4.618921138899168e-06, 4.6827317419229075e-06, 4.54903238278348e-06, 4.556341991701629e-06, 4.529045781964669e-06, 4.460462150746025e-06, 4.363975676824339e-06, 4.3109639591421e-06, 4.248049663146958e-06, 4.15251679442008e-06, 4.00755061491509e-06, 3.7691477245971328e-06, 3.5547336665331386e-06, 3.332451569804107e-06, 3.0715416414750507e-06, 2.942733544841758e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.269887292844942e-05, "first_to_last_ratio": 2.3568639793204684, "imbalance_abs_log_ratio": 0.8573319130483905, "last_quartile_mean": 1.3873890566173941e-05, "mean": 2.12624314031018e-05, "normalized": [ 1.6816001107908207, 1.968609461712434, 1.7075260440141944, 1.4905139409933708, 1.4972595154646264, 1.4034185225085787, 1.3238781474315933, 1.2301608589864579, 1.121553773570522, 1.0499501635664128, 1.001131281922829, 0.9633193641679035, 0.9560899219831485, 0.9203740514079343, 0.8930538331784411, 0.8703809179546808, 0.8841528402061924, 0.8558056863273277, 0.8550540488218812, 0.8483642525948266, 0.8350365031100208, 0.8178976965352276, 0.8078428125234317, 0.7969682791900239, 0.7797478584358322, 0.7532962238158083, 0.7097589943815636, 0.6713244476982704, 0.629948240254275, 0.5812766747709892, 0.5573943041371163, 0.5373112275432649 ], "peak_layer": 2, "peak_normalized": 1.968609461712434, "population_cv": 0.35388972104860866, "rest_mean": 2.2020403102268808e-05, "spike_contrast": 0.7797034481094922, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.716938422759995e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.5754907003138214e-05, 4.185742363915779e-05, 3.630615537986159e-05, 3.169195042573847e-05, 3.183537774020806e-05, 2.9840090064681135e-05, 2.8148868295829743e-05, 2.6156210878980346e-05, 2.3846960175433196e-05, 2.2324493329506367e-05, 2.128648520738352e-05, 2.048251189989969e-05, 2.0328796381363645e-05, 1.9569390133256093e-05, 1.8988495867233723e-05, 1.8506414562580176e-05, 1.8799239114741795e-05, 1.819650969991926e-05, 1.8180528059019707e-05, 1.8038286725641228e-05, 1.775490636646282e-05, 1.739049366733525e-05, 1.7176702385768294e-05, 1.6945483366725966e-05, 1.6579335351707414e-05, 1.6016909285099246e-05, 1.5091201930772513e-05, 1.4273990018409677e-05, 1.3394231245911215e-05, 1.2359355423541274e-05, 1.1851558156195097e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02899768785573542, "first_to_last_ratio": 2.3568640590123793, "imbalance_abs_log_ratio": 0.8573319468610798, "last_quartile_mean": 0.012303504627197981, "mean": 0.018855736823752522, "normalized": [ 1.6816000706700753, 1.9686094816861437, 1.707526129634271, 1.4905139760038595, 1.4972595403829805, 1.403418590801014, 1.323878107307901, 1.2301608352967786, 1.12155375183925, 1.0499501677926542, 1.0011313114380156, 0.963319363752492, 0.9560898565462423, 0.9203740066049206, 0.8930539078006676, 0.8703809048152793, 0.8841528764919796, 0.8558057204161084, 0.8550539741588212, 0.848364321524986, 0.8350365815065012, 0.8178976558963745, 0.8078427780491525, 0.7969682399430816, 0.7797478175310808, 0.7532962276748001, 0.7097590061776181, 0.6713244229257764, 0.6299482002622888, 0.5812766802470385, 0.5573943141138397, 0.5373111807080079 ], "peak_layer": 2, "peak_normalized": 1.9686094816861437, "population_cv": 0.3538897326937565, "rest_mean": 0.019527913761083728, "spike_contrast": 0.7797034311216169, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01522598136216402, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.03170780837535858, 0.037119582295417786, 0.032196663320064545, 0.028104739263653755, 0.02823193185031414, 0.02646249160170555, 0.02496269717812538, 0.02319558896124363, 0.021147722378373146, 0.019797584041953087, 0.018877068534493446, 0.018164096400141716, 0.01802777871489525, 0.01735433004796505, 0.01683918945491314, 0.0164116732776165, 0.016671353951096535, 0.01613684743642807, 0.016122672706842422, 0.015996534377336502, 0.01574523001909256, 0.015422062948346138, 0.015232470817863941, 0.015027423389256, 0.014702719636261463, 0.014203955419361591, 0.013383029028773308, 0.012658316642045975, 0.011878137476742268, 0.010960400104522705, 0.010510080493986607, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.231474328733384e-06, "first_to_last_ratio": 1.3582476774197694, "imbalance_abs_log_ratio": 0.3061953964681102, "last_quartile_mean": 9.066640416222072e-07, "mean": 1.0095540616816834e-06, "normalized": [ 2.2510689330098104, 1.3453781719433024, 1.2282097485564385, 1.0527316026565288, 1.0056637182516928, 0.969318760716823, 0.9573356047524155, 0.9488541987585868, 0.9217951398143368, 0.8906270200688119, 0.8763270628110001, 0.8694286288822738, 0.8685312885606712, 0.8765222738855435, 0.8891328417344724, 0.8990263889886259, 0.900299568341834, 0.9275418452948997, 0.9582330013794911, 0.9876130845276129, 1.0001282145999768, 1.0337196092881593, 1.0681495050787642, 1.0896942312291085, 1.0818926575150707, 1.0552217685009913, 0.9988175358018173, 0.923772869549939, 0.860068575983845, 0.7959012825140968, 0.7583091587544984, 0.710685708248561 ], "peak_layer": 1, "peak_normalized": 2.2510689330098104, "population_cv": 0.25617417026860584, "rest_mean": 9.993245039704899e-07, "spike_contrast": 1.065513423409031, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0647936733221285e-06, "top_five_layers": [ 1, 2, 3, 24, 25 ], "top_five_spike_overlap": 2 }, "values": [ 2.2725757844455075e-06, 1.3582319979832391e-06, 1.2399441402521916e-06, 1.0627894653225667e-06, 1.0152718914469006e-06, 9.785796919459244e-07, 9.664820481702918e-07, 9.579196103004506e-07, 9.306020274379989e-07, 8.991361255539232e-07, 8.846995456224249e-07, 8.777352036304364e-07, 8.768292900640517e-07, 8.848966217556153e-07, 8.976276717476139e-07, 9.076157425624842e-07, 9.089010859497648e-07, 9.364036372971896e-07, 9.673880185800954e-07, 9.970488008548273e-07, 1.0096835012518568e-06, 1.043595830196864e-06, 1.0783546713355463e-06, 1.1001052371284459e-06, 1.09222912669793e-06, 1.0653034223651048e-06, 1.008360300147615e-06, 9.325986525254848e-07, 8.682857242092723e-07, 8.035053724597674e-07, 7.655540912310244e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.523178955016192e-05, "first_to_last_ratio": 2.651853802016533, "imbalance_abs_log_ratio": 0.9752589433592965, "last_quartile_mean": 9.514773978480662e-06, "mean": 1.5758304414248414e-05, "normalized": [ 1.768014772712438, 2.0202664913584756, 1.7747876589600569, 1.5702354129512541, 1.5745618645822212, 1.4685789787379961, 1.3665832942020837, 1.266364662649249, 1.14652665049396, 1.0704311524660057, 1.0164222471522504, 0.9740486236599476, 0.9677569672847843, 0.9245129976091487, 0.891368391145493, 0.8644143239142155, 0.883928662735923, 0.8455178269951987, 0.8460475954048016, 0.8329649879043416, 0.8145835420025702, 0.783683597592109, 0.7613793029015486, 0.7366657999397114, 0.716423036769208, 0.6895960585495816, 0.6528852259142993, 0.6205906498609965, 0.5818984522075571, 0.54251886575813, 0.5218602063941854, 0.5045816991902581 ], "peak_layer": 2, "peak_normalized": 2.0202664913584756, "population_cv": 0.3941497713845743, "rest_mean": 1.645124066945519e-05, "spike_contrast": 0.7304281104125244, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2016448636131827e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.786091499729082e-05, 3.183597436873242e-05, 2.7967644200543873e-05, 2.4744247639318928e-05, 2.481242518115323e-05, 2.314231460331939e-05, 2.1535035557462834e-05, 1.9955759853473864e-05, 1.8067315977532417e-05, 1.6868179955054075e-05, 1.60170911840396e-05, 1.5349354725913145e-05, 1.5250208889483474e-05, 1.4568757251254283e-05, 1.404645445290953e-05, 1.362170405627694e-05, 1.3929216947872192e-05, 1.3323927305464167e-05, 1.3332275557331741e-05, 1.3126115845807362e-05, 1.2836455425713211e-05, 1.234952469530981e-05, 1.1998046829830855e-05, 1.1608603927015793e-05, 1.1289612302789465e-05, 1.086686461349018e-05, 1.0288364137522876e-05, 9.779456377145834e-06, 9.169732948066667e-06, 8.549177437089384e-06, 8.22363199404208e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9022438777938078e-05, "first_to_last_ratio": 2.5706811468129205, "imbalance_abs_log_ratio": 0.9441709014612467, "last_quartile_mean": 7.3997659342239785e-06, "mean": 1.1811503938474743e-05, "normalized": [ 1.7838575112897632, 2.0081895041574866, 1.8013630165639003, 1.6167453942621992, 1.616253821452643, 1.484346910873369, 1.334088872746376, 1.2391628829890091, 1.1149528907165038, 1.0471548053789776, 0.9987803140821989, 0.9467453609714046, 0.9424805202441363, 0.9017388056265639, 0.868090708413055, 0.8437486142496254, 0.8736503938091038, 0.8327134052809684, 0.8458575878838563, 0.8280086592496597, 0.8103372943527495, 0.7723135523278634, 0.7500419314219764, 0.7274729336065492, 0.7078485216009954, 0.6848456245402912, 0.657709866043606, 0.6402184904077136, 0.6083853394516672, 0.5800023997525418, 0.5702454032899359, 0.5626486629633098 ], "peak_layer": 2, "peak_normalized": 2.0081895041574866, "population_cv": 0.39403850931287454, "rest_mean": 1.2350452262008158e-05, "spike_contrast": 0.7207171691012223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.901182991394307e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.1070040020276792e-05, 2.3719738237559795e-05, 2.1276806364767253e-05, 1.9096194591838866e-05, 1.9090388377662748e-05, 1.7532369383843616e-05, 1.5757595974719152e-05, 1.4636377272836398e-05, 1.3169270459911786e-05, 1.2368473107926548e-05, 1.1797097613452934e-05, 1.1182486559846438e-05, 1.1132112376799341e-05, 1.065089145413367e-05, 1.025345682137413e-05, 9.965940080292057e-06, 1.031912506732624e-05, 9.835597666096874e-06, 9.990850230678916e-06, 9.780027539818548e-06, 9.571302143740468e-06, 9.122184565057978e-06, 8.859123227011878e-06, 8.592549420427531e-06, 8.360755600733683e-06, 8.089056791504845e-06, 7.76854267314775e-06, 7.5619432209350634e-06, 7.185945833043661e-06, 6.850700629001949e-06, 6.735455826856196e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.828027426308836e-05, "first_to_last_ratio": 2.3127230006738477, "imbalance_abs_log_ratio": 0.838425618478933, "last_quartile_mean": 2.9523758030336467e-05, "mean": 4.4895063126659807e-05, "normalized": [ 1.6579137880786858, 1.9150708402023993, 1.6745616807855777, 1.4835198878876907, 1.4915155698146907, 1.3960120954471311, 1.3299713054901032, 1.2185230612071285, 1.1078646449578662, 1.0477053876101448, 0.999765654013293, 0.9692189920089408, 0.955488933285625, 0.9355193384387992, 0.9031699366923724, 0.8872109049801036, 0.8933262176117003, 0.8675434009211797, 0.864331741509242, 0.8580824004003949, 0.8343570910048359, 0.8204903297173597, 0.8195590181762232, 0.8083415477202021, 0.7958344380770943, 0.7671025897223802, 0.7201771453938138, 0.6826439856557369, 0.6331928257241357, 0.58535097991478, 0.5516010454074354, 0.5250332221429341 ], "peak_layer": 2, "peak_normalized": 1.9150708402023993, "population_cv": 0.34285892534032564, "rest_mean": 4.642717794922646e-05, "spike_contrast": 0.788797525553888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.662164308479987e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.443214417435229e-05, 8.597722626291215e-05, 7.517955236835405e-05, 6.660271901637316e-05, 6.696168566122651e-05, 6.267405115067959e-05, 5.9709145716624334e-05, 5.470566975418478e-05, 4.9737653171177953e-05, 4.703679951489903e-05, 4.4884542148793116e-05, 4.351314782979898e-05, 4.2896735976682976e-05, 4.200019975542091e-05, 4.0547871321905404e-05, 3.983138958574273e-05, 4.010593693237752e-05, 3.89484157494735e-05, 3.8804228097433224e-05, 3.8523663533851504e-05, 3.745851427083835e-05, 3.683596514747478e-05, 3.6794153857044876e-05, 3.629054481280036e-05, 3.5729037335840985e-05, 3.4439119190210477e-05, 3.233239840483293e-05, 3.064734482904896e-05, 2.8427231882233173e-05, 2.6279369194526225e-05, 2.4764163754298352e-05, 2.357139965170063e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.4383448545868305e-06, "first_to_last_ratio": 5.042948145238695, "imbalance_abs_log_ratio": 1.6179908605191557, "last_quartile_mean": 1.276702569441568e-06, "mean": 3.687052599232743e-06, "normalized": [ 3.304965820363612, 2.227169292436081, 1.6521910965438824, 1.3947865539668498, 0.5800793227551304, 1.2714767915022098, 1.750193241091026, 1.7887674026051423, 1.1778703519658833, 1.1179830646270124, 0.8417156564271726, 0.7655624105040819, 0.5268426976300161, 1.047779560839775, 0.8380697743291855, 0.7915788420213868, 0.7099070375965003, 1.3022194692601015, 1.0739447359104763, 1.3540969285822078, 0.6285621976492703, 1.167508624430284, 0.9740363058232884, 0.9425613189042933, 0.35040916273624656, 0.44668469186889276, 0.3295723924580137, 0.4329866871315858, 0.2917160480921282, 0.22501238083060152, 0.3167851028104899, 0.37696503630717276 ], "peak_layer": 1, "peak_normalized": 3.304965820363612, "population_cv": 0.641995540013263, "rest_mean": 3.814996382010981e-06, "spike_contrast": 0.7853627820875961, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9961561722302575e-06, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 1.218558281834703e-05, 8.211690328607801e-06, 6.091715476941317e-06, 5.142651389178354e-06, 2.1387829747254727e-06, 4.688001808972331e-06, 6.4530545387242455e-06, 6.5952795011980925e-06, 4.342869942774996e-06, 4.122062364331214e-06, 3.1034498988447012e-06, 2.8226688755239593e-06, 1.942496737683541e-06, 3.863218353217235e-06, 3.090007339778822e-06, 2.918592826972599e-06, 2.6174645881837932e-06, 4.80135167890694e-06, 3.959690729971044e-06, 4.992626600142103e-06, 2.3175418846221874e-06, 4.3046657083323225e-06, 3.5913230931328144e-06, 3.475273160802317e-06, 1.2919770142616471e-06, 1.646949954192678e-06, 1.215150746247673e-06, 1.5964446902216878e-06, 1.0755724133559852e-06, 8.296324836010172e-07, 1.1680033367156284e-06, 1.3898899169362267e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.5929461344276206e-05, "first_to_last_ratio": 5.201390647867945, "imbalance_abs_log_ratio": 1.6489260221160402, "last_quartile_mean": 4.985101696775018e-06, "mean": 1.4714017034123117e-05, "normalized": [ 3.407292787305506, 2.2558164668132807, 1.6137698282608655, 1.433949322998548, 0.5790123126574623, 1.305711460245382, 1.7258967842855322, 1.7763796775082434, 1.1315333512208903, 1.0856488588758444, 0.8714057435432361, 0.7593394863607054, 0.5216263168993628, 1.0262173986737884, 0.8250528440437026, 0.7676154211606976, 0.723733557926003, 1.3830217582174593, 1.0774149558606307, 1.3018941088272185, 0.646511650324857, 1.1377981887421273, 0.9823485776546648, 0.9506130966721329, 0.33641925298333475, 0.4509311419254927, 0.32185944273605893, 0.4000452689662174, 0.290655292962286, 0.22712751082346105, 0.30970536495859485, 0.3736527695664126 ], "peak_layer": 1, "peak_normalized": 3.407292787305506, "population_cv": 0.6554716761859383, "rest_mean": 1.522972111497775e-05, "spike_contrast": 0.7832851900207317, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1929214997508097e-05, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 5.0134964112658054e-05, 3.3192121918546036e-05, 2.374503674218431e-05, 2.1099154764669947e-05, 8.519597031408921e-06, 1.921226066770032e-05, 2.539487468311563e-05, 2.6137680833926424e-05, 1.6649401004542597e-05, 1.59742558025755e-05, 1.2821878954127897e-05, 1.1172934136993717e-05, 7.675218512304127e-06, 1.5099780284799635e-05, 1.2139841601310764e-05, 1.1294706382614095e-05, 1.0649027899489738e-05, 2.03498057089746e-05, 1.5853102013352327e-05, 1.9156092093908228e-05, 9.512783435638994e-06, 1.674158193054609e-05, 1.4454293705057353e-05, 1.3987337297294289e-05, 4.9500786190037616e-06, 6.635008503508288e-06, 4.735845323011745e-06, 5.886272901989287e-06, 4.276706931705121e-06, 3.3419580631743884e-06, 4.557010015560081e-06, 5.497933216247475e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.022994505590759218, "first_to_last_ratio": 5.20139065184949, "imbalance_abs_log_ratio": 1.6489260228815172, "last_quartile_mean": 0.004420838027726859, "mean": 0.01304853729379829, "normalized": [ 3.4072926237702874, 2.255816491845768, 1.613769818466143, 1.4339493120942854, 0.5790123351120128, 1.3057114456299215, 1.725896924613148, 1.7763798383286113, 1.13153331687909, 1.0856488688373112, 0.8714057478512427, 0.7593395234826519, 0.5216262984837251, 1.0262174043007157, 0.8250528027479058, 0.7676154484189387, 0.7237335326830917, 1.3830217401378855, 1.0774149116451128, 1.3018940938870247, 0.6465115988335997, 1.1377982160374271, 0.9823485499546105, 0.9506130843152019, 0.3364192535702373, 0.45093116963769325, 0.3218594440675733, 0.4000452442436667, 0.2906553022018551, 0.22712752175948964, 0.30970535698369017, 0.3736527791800823 ], "peak_layer": 1, "peak_normalized": 3.4072926237702874, "population_cv": 0.6554716682951498, "rest_mean": 0.013505868855173941, "spike_contrast": 0.7832851759342461, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.010578946862369775, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 0.04446018487215042, 0.029435105621814728, 0.021057335659861565, 0.018710941076278687, 0.007555264048278332, 0.01703762449324131, 0.022520430386066437, 0.023179158568382263, 0.014764854684472084, 0.014166129752993584, 0.01137057039886713, 0.009908270090818405, 0.0068064602091908455, 0.013390636071562767, 0.010765732266008854, 0.010016258805990219, 0.009443663991987705, 0.018046410754323006, 0.014058688655495644, 0.016987813636660576, 0.008436030708253384, 0.014846602454781532, 0.012818211689591408, 0.01240411028265953, 0.0043897791765630245, 0.005883992183953524, 0.004199794959276915, 0.005220005288720131, 0.0037926265504211187, 0.0029636819381266832, 0.004041201900690794, 0.004875622224062681 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.464145257619066e-06, "first_to_last_ratio": 2.649593113901558, "imbalance_abs_log_ratio": 0.9744060862854225, "last_quartile_mean": 9.300089303110326e-07, "mean": 1.2621179932281734e-06, "normalized": [ 9.98073298735891, 2.6081595638248576, 0.448046131778418, 0.5712169168045845, 0.31851900459136234, 0.5125315286968626, 0.5097468218730781, 0.6701588608628778, 0.5743479667542774, 0.6374764102214231, 0.546790403537672, 0.5390341640801587, 0.35740547847929505, 0.6625310250792219, 0.6002964531361203, 0.6405992632336756, 0.4466467073820306, 0.8660901213820976, 0.747225340722879, 0.8852501473817866, 0.3909436983971323, 0.8018997241352653, 0.8266134004412924, 0.9628283052981523, 0.5300152809900267, 0.7412196878662661, 0.7889073080638102, 1.074944484309473, 0.6781360573263348, 0.7166665274120905, 0.6792494446365528, 0.6857707839420146 ], "peak_layer": 1, "peak_normalized": 9.98073298735891, "population_cv": 1.6572947531819389, "rest_mean": 1.3316606755240543e-06, "spike_contrast": 0.6657758429950736, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.865875088304165e-07, "top_five_layers": [ 1, 2, 28, 24, 20 ], "top_five_spike_overlap": 1 }, "values": [ 1.259686268895166e-05, 3.2918051147134975e-06, 5.654870847138227e-07, 7.209431487353868e-07, 4.020085668798856e-07, 6.468752644650522e-07, 6.433606358768884e-07, 8.45819556616334e-07, 7.248949032145902e-07, 8.045704475989623e-07, 6.901140068293898e-07, 6.803247174502758e-07, 4.51087885267043e-07, 8.361923278243921e-07, 7.576449547741504e-07, 8.08511856575933e-07, 5.637208460029797e-07, 1.0931079259535181e-06, 9.430865475223982e-07, 1.1172901395184454e-06, 4.934170760861889e-07, 1.012092070595827e-06, 1.0432836461404804e-06, 1.2152029285061872e-06, 6.68941822823399e-07, 9.355067049909849e-07, 9.956941084965365e-07, 1.356706775368366e-06, 8.558877198083792e-07, 9.045177193911513e-07, 8.572929459660372e-07, 8.655236456434068e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.6010561125767708e-05, "first_to_last_ratio": 4.917014092262474, "imbalance_abs_log_ratio": 1.5927014544542644, "last_quartile_mean": 3.2561552245624625e-06, "mean": 9.725274459526645e-06, "normalized": [ 2.0495551017102773, 2.324610334145404, 1.6789860752319785, 1.469067342376247, 0.683331640000088, 1.4508358782825064, 1.8574881471610558, 1.6563953320037643, 1.2394147680445438, 1.1757596976660154, 0.9385611021097731, 0.8217156066495895, 0.5863388014820525, 1.1122247987743321, 0.8990502135337656, 0.7981592831866082, 0.7526244364476551, 1.397025330921264, 1.1617275022678442, 1.3251897151639205, 0.6397940246687674, 1.2096957514764108, 1.0918700868694384, 1.0020693481814167, 0.3582271576029711, 0.48201484763579705, 0.332690365953382, 0.356368076645877, 0.30908080698932155, 0.24694487721404904, 0.26695462213676374, 0.32622892746712 ], "peak_layer": 2, "peak_normalized": 2.324610334145404, "population_cv": 0.5441079654169187, "rest_mean": 9.97681458843376e-06, "spike_contrast": 0.83864019815785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.366957763428217e-06, "top_five_layers": [ 2, 1, 7, 3, 8 ], "top_five_spike_overlap": 0 }, "values": [ 1.9932485884055495e-05, 2.2607473511015996e-05, 1.6328600395354442e-05, 1.42870831041364e-05, 6.645587745879311e-06, 1.4109777112025768e-05, 1.8064582036458887e-05, 1.6108899217215367e-05, 1.2053648788423743e-05, 1.143458575825207e-05, 9.127764315053355e-06, 7.991409802343696e-06, 5.7023057706828695e-06, 1.0816691428772174e-05, 8.743510079511907e-06, 7.762318091408815e-06, 7.319479209400015e-06, 1.3586454770120326e-05, 1.1298118806735147e-05, 1.2887833690911066e-05, 6.222172487468924e-06, 1.176462319563143e-05, 1.0618736268952489e-05, 9.745399438543245e-06, 3.4838574265450006e-06, 4.6877266868250445e-06, 3.2355051189369988e-06, 3.465777353994781e-06, 3.0058956781431334e-06, 2.4016067072807346e-06, 2.5962069685192546e-06, 3.1726658562547527e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.92899242646672e-06, "first_to_last_ratio": 4.574804953054558, "imbalance_abs_log_ratio": 1.520564064606545, "last_quartile_mean": 2.1703640982195793e-06, "mean": 6.422408389994416e-06, "normalized": [ 1.2694651099378356, 2.2660081023317744, 1.6776221325435599, 1.4243775268645795, 0.778686696628982, 1.544586331363982, 1.9675189416357637, 1.4396717003002808, 1.3121814531183145, 1.2749800755885756, 1.028924014649809, 0.867611835816526, 0.6467747339714943, 1.1792708375805625, 0.9308967888836265, 0.8222106571344964, 0.7916222394567357, 1.4570716548377307, 1.2330890513227566, 1.308271526002021, 0.6046831152920485, 1.2497078701132496, 1.207744340136328, 1.0135338985983424, 0.3691120245165777, 0.5161808316774584, 0.34931346918180795, 0.3250152051676304, 0.32914583570336414, 0.26959120942422543, 0.2525212479205583, 0.29260954229900277 ], "peak_layer": 2, "peak_normalized": 2.2660081023317744, "population_cv": 0.5120965787375209, "rest_mean": 6.5544765961762836e-06, "spike_contrast": 0.8710443912398678, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7092400766123316e-06, "top_five_layers": [ 2, 7, 3, 6, 18 ], "top_five_spike_overlap": 0 }, "values": [ 8.153023372869939e-06, 1.4553229448210914e-05, 1.0774374459288083e-05, 9.147934179054573e-06, 5.001043973607011e-06, 9.919964213622734e-06, 1.2636210158234462e-05, 9.24615960684605e-06, 8.427365173702128e-06, 8.188442734535784e-06, 6.608170224353671e-06, 5.572157533606514e-06, 4.153851477894932e-06, 7.5737589213531464e-06, 5.978599347145064e-06, 5.280572622723412e-06, 5.084121312393108e-06, 9.357909220852889e-06, 7.919401468825527e-06, 8.402254024986178e-06, 3.883521912939614e-06, 8.026134310057387e-06, 7.756627383059822e-06, 6.509328613901744e-06, 2.3705881631030934e-06, 3.3151241041196045e-06, 2.2434337552112993e-06, 2.087380380544346e-06, 2.1139089767530095e-06, 1.731424845274887e-06, 1.6217945812968537e-06, 1.8792579794535413e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.949699314238387e-05, "first_to_last_ratio": 5.262607641337885, "imbalance_abs_log_ratio": 1.6606266532103107, "last_quartile_mean": 9.405412015439651e-06, "mean": 2.8804784136582384e-05, "normalized": [ 2.499900864670067, 2.416668640808701, 1.6691856383541526, 1.5215442251975408, 0.6043399354947968, 1.3939313584258168, 1.817946933469592, 1.823363590450555, 1.1883363380697656, 1.0810267152424702, 0.8648382834598256, 0.8031803638827597, 0.533176912017834, 1.0261239559732709, 0.8554824003261805, 0.7869370867194827, 0.7081002132573654, 1.3561635382832782, 1.1101608226630673, 1.398379059233539, 0.6739882703135641, 1.211767597856563, 0.9998718364198766, 1.043404895274106, 0.34963454859286996, 0.4704717001014101, 0.31939467298460794, 0.36230101093546613, 0.28800141494502707, 0.22353921217823322, 0.2657184225315432, 0.3331195418666723 ], "peak_layer": 1, "peak_normalized": 2.499900864670067, "population_cv": 0.5921886777524846, "rest_mean": 2.957433365803876e-05, "spike_contrast": 0.8334665120685795, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4649216720717958e-05, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 7.200910476967692e-05, 6.961161852814257e-05, 4.8080531996674836e-05, 4.382775296107866e-05, 1.7407881387043744e-05, 4.01518918806687e-05, 5.2365568990353495e-05, 5.2521594625432044e-05, 3.4229771699756384e-05, 3.113874117843807e-05, 2.4911480068112724e-05, 2.3135437004384585e-05, 1.5358045857283287e-05, 2.9557279049186036e-05, 2.4641985874040984e-05, 2.2667552912025712e-05, 2.0396673789946362e-05, 3.906399797415361e-05, 3.197794285370037e-05, 4.028000694233924e-05, 1.941408663697075e-05, 3.490470407996327e-05, 2.880109241232276e-05, 3.0055052775423974e-05, 1.0071147698909044e-05, 1.3551835763792042e-05, 9.200094609695952e-06, 1.0436002412461676e-05, 8.295818588521797e-06, 6.438998752855696e-06, 7.65396180213429e-06, 9.595436495146714e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.6960831434053034e-06, "first_to_last_ratio": 0.38455700861487224, "imbalance_abs_log_ratio": 0.9556632341481542, "last_quartile_mean": 7.0108802674440085e-06, "mean": 4.081287496404684e-06, "normalized": [ 1.6016557623725205, 0.5624254649150265, 0.3792516557450063, 0.6472346627621453, 0.3825774509297101, 0.309883335456179, 0.41924156145633357, 0.9824999654437843, 0.9295888075476517, 0.6942195909151134, 0.6392271159962603, 0.6622046687372564, 0.5739498407816973, 0.6102779288338825, 0.725372269896936, 0.7725414351920691, 0.5151773567621832, 0.6691928665493493, 0.6340666932552725, 0.5067672404198594, 0.8579891398085714, 1.216818523393502, 1.4001261788695554, 1.56522304857102, 1.4389835581991173, 1.8374458775865694, 2.199543673756739, 1.6451073151800026, 2.0248581159734593, 1.9770863776726855, 1.3258318688286215, 1.2936306481919198 ], "peak_layer": 27, "peak_normalized": 2.199543673756739, "population_cv": 0.5384151707957241, "rest_mean": 3.857702406801562e-06, "spike_contrast": 1.3709318196595643, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.288646980261546e-06, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 6.5368176365154795e-06, 2.295420017617289e-06, 1.5478350405828678e-06, 2.6415507363708457e-06, 1.5614085668858024e-06, 1.2647229823414818e-06, 1.7110453427449102e-06, 4.009864824183751e-06, 3.793919177041971e-06, 2.833309736161027e-06, 2.6088696358783636e-06, 2.7026476345781703e-06, 2.3424543087458005e-06, 2.4907196802814724e-06, 2.960452775369049e-06, 3.1529636999039212e-06, 2.1025869045843137e-06, 2.7311684789310675e-06, 2.587808467069408e-06, 2.068262801913079e-06, 3.501700348351733e-06, 4.966186224919511e-06, 5.7143174672091845e-06, 6.388125257217325e-06, 5.872905603609979e-06, 7.499144885514397e-06, 8.976970093499403e-06, 6.714155915688025e-06, 8.264028110716026e-06, 8.06905791250756e-06, 5.411101028585108e-06, 5.279678589431569e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.0839275830676343e-05, "first_to_last_ratio": 0.38713025500084114, "imbalance_abs_log_ratio": 0.9489940663518645, "last_quartile_mean": 2.7999040867143776e-05, "mean": 1.6318590368769037e-05, "normalized": [ 1.6422297380818347, 0.577857691983563, 0.3782290994995378, 0.6502872006236952, 0.3822780102727073, 0.31178153988993046, 0.41970720095668634, 0.9514590030920431, 0.920750817583778, 0.6902360262942715, 0.6326124977684876, 0.6666642010329482, 0.573439016269761, 0.6073224038931471, 0.7243939579070396, 0.7687525976401163, 0.5168423803555005, 0.6726938023393403, 0.6400255756173949, 0.5037402338382693, 0.8679547154047897, 1.2120933210153013, 1.403510673963058, 1.5589326681275335, 1.4338206424993105, 1.8383886875461704, 2.207829035366123, 1.646625228594747, 2.022240419628589, 1.965711109970758, 1.3282014630518373, 1.283389039891731 ], "peak_layer": 27, "peak_normalized": 2.207829035366123, "population_cv": 0.5389668648883295, "rest_mean": 1.542631884553379e-05, "spike_contrast": 1.3701814934519443, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1136856594239363e-05, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 2.6798874387168325e-05, 9.429822966922075e-06, 6.1721657402813435e-06, 1.061177044903161e-05, 6.238238256628392e-06, 5.087835234007798e-06, 6.849029887234792e-06, 1.5526469724136405e-05, 1.5025355423858855e-05, 1.1263678970863111e-05, 1.0323344213247765e-05, 1.0879020010179374e-05, 9.357716407976113e-06, 9.910645530908369e-06, 1.18210882646963e-05, 1.254495873581618e-05, 8.434139090240933e-06, 1.097741460398538e-05, 1.044431519403588e-05, 8.220330528274644e-06, 1.4163797459332272e-05, 1.9779654394369572e-05, 2.2903315766598098e-05, 2.5439583623665385e-05, 2.339793172723148e-05, 2.9999911930644885e-05, 3.602865763241425e-05, 2.687060259631835e-05, 3.300011303508654e-05, 3.2077634386951104e-05, 2.1674375602742657e-05, 2.094310002576094e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.009612377849407494, "first_to_last_ratio": 0.38713024225946485, "imbalance_abs_log_ratio": 0.9489940992642404, "last_quartile_mean": 0.024829829344525933, "mean": 0.014471489033894613, "normalized": [ 1.6422296302631012, 0.5778576663317349, 0.37822910199946935, 0.6502872213280481, 0.38227800767762105, 0.3117815365380894, 0.4197072091098399, 0.9514589510980404, 0.9207507998752875, 0.6902360121427672, 0.6326125119311165, 0.6666641983580384, 0.573439005071423, 0.607322401384804, 0.7243939498038978, 0.7687525851502272, 0.5168424285086333, 0.6726938110990978, 0.6400255788101445, 0.5037402548123486, 0.8679547482284273, 1.2120933573235502, 1.4035107155843847, 1.5589326526958214, 1.4338205694791435, 1.8383887153176988, 2.2078290833379297, 1.6466252521885574, 2.022240548084217, 1.9657110301441079, 1.3282014367981623, 1.28338902952427 ], "peak_layer": 27, "peak_normalized": 2.2078290833379297, "population_cv": 0.5389668665215603, "rest_mean": 0.01368021371739882, "spike_contrast": 1.3701814993673933, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.018744375742971896, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 0.023765508085489273, 0.008362460881471634, 0.005473538301885128, 0.009410624392330647, 0.0055321319960057735, 0.004511943086981773, 0.006073788274079561, 0.013769027777016163, 0.013324635103344917, 0.009988742880523205, 0.009154845029115677, 0.009647623635828495, 0.008298516273498535, 0.008788859471678734, 0.010483059100806713, 0.011124994605779648, 0.007479479536414146, 0.009734881110489368, 0.00926212314516306, 0.007289871573448181, 0.012560597620904446, 0.017540795728564262, 0.020310889929533005, 0.022560076788067818, 0.020749518647789955, 0.026604222133755684, 0.03195057436823845, 0.02382911927998066, 0.02926483191549778, 0.02844676561653614, 0.019221052527427673, 0.018572550266981125 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.596527093449822e-07, "first_to_last_ratio": 0.3487620963761673, "imbalance_abs_log_ratio": 1.053365261498487, "last_quartile_mean": 1.8914116992618801e-06, "mean": 1.0618384331806396e-06, "normalized": [ 1.9623284636711533, 0.5051262881892306, 0.3546331093387114, 0.6524291934355505, 0.33242707700757573, 0.26547608492472724, 0.31949906663212685, 0.5779720956168373, 0.775638331440552, 0.5998773794374831, 0.5826887294747466, 0.6213115726756598, 0.5572960370183113, 0.6050525236179888, 0.7278412668111575, 0.7634285200712856, 0.5211325543229903, 0.6528362049921996, 0.6569013419877625, 0.48939266573007467, 0.8730686968435768, 1.2548058690120942, 1.4927712810480218, 1.6059753116097122, 1.4417731559107498, 1.8108704859544356, 2.3560676855554643, 1.7166277442710838, 2.056050650260877, 2.0539911278871554, 1.42644194116846, 1.3882675440822447 ], "peak_layer": 27, "peak_normalized": 2.3560676855554643, "population_cv": 0.600205151665452, "rest_mean": 9.962249070785792e-07, "spike_contrast": 1.4215178360523206, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.416151474131766e-06, "top_five_layers": [ 27, 29, 30, 1, 26 ], "top_five_spike_overlap": 0 }, "values": [ 2.0836757812503492e-06, 5.363625064092048e-07, 3.765630651741958e-07, 6.927743925189134e-07, 3.52983846596544e-07, 2.818927100634028e-07, 3.3925638831533433e-07, 6.137129844319134e-07, 8.236025905716815e-07, 6.369728566824051e-07, 6.187212875374826e-07, 6.597325068469218e-07, 5.917583507653035e-07, 6.424680236705171e-07, 7.728498303549713e-07, 8.106377435979084e-07, 5.533585749617487e-07, 6.932065730325121e-07, 6.975230917305453e-07, 5.19655941388919e-07, 9.270578971154464e-07, 1.332401097897673e-06, 1.5850819181650877e-06, 1.7052863086064463e-06, 1.5309301488741767e-06, 1.9228518794989213e-06, 2.50176321969775e-06, 1.8227813143312233e-06, 2.183193601013045e-06, 2.181006721002632e-06, 1.5146508758334676e-06, 1.4741158338438254e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 7.227550952393358e-06, "first_to_last_ratio": 0.3980164784812622, "imbalance_abs_log_ratio": 0.9212618713355372, "last_quartile_mean": 1.8158923921873793e-05, "mean": 1.0911241304256691e-05, "normalized": [ 1.6660127430322156, 0.5876271935840073, 0.29483709953854387, 0.5894374739528777, 0.41635452404712836, 0.33657065533584085, 0.43430341472813805, 0.9740163595711268, 0.9875415328473797, 0.7356703350916547, 0.6756170470367248, 0.7197599061272355, 0.6198568191216928, 0.6512295663928718, 0.7665848858812202, 0.8116271754128533, 0.5305907755231244, 0.6901913960659463, 0.6428575406488269, 0.5176364098703187, 0.8723657576961961, 1.2199632610317463, 1.415147460346549, 1.5302808917079092, 1.4588334149411595, 1.8140206258329823, 2.1102391081606, 1.5869337150428715, 1.998597539089171, 1.88124956086161, 1.2686189428642605, 1.195426868615217 ], "peak_layer": 27, "peak_normalized": 2.1102391081606, "population_cv": 0.5159943803714161, "rest_mean": 1.0306438963636066e-05, "spike_contrast": 1.375564731293613, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4177173943608068e-05, "top_five_layers": [ 27, 29, 30, 26, 1 ], "top_five_spike_overlap": 0 }, "values": [ 1.81782670551911e-05, 6.411742106138263e-06, 3.2170387385122012e-06, 6.431494512071367e-06, 4.542944679997163e-06, 3.672403636301169e-06, 4.738789357361384e-06, 1.0627727533574216e-05, 1.0775303962873295e-05, 8.027076546568424e-06, 7.371820629487047e-06, 7.85347401688341e-06, 6.763407327525783e-06, 7.105722943379078e-06, 8.364392670046072e-06, 8.855859960021917e-06, 5.789403985545505e-06, 7.530844868597342e-06, 7.014373750280356e-06, 5.648055775964167e-06, 9.51859328779392e-06, 1.3311313523445278e-05, 1.5441015420947224e-05, 1.66972640727181e-05, 1.591768341313582e-05, 1.979321677936241e-05, 2.3025328118819743e-05, 1.7315416698693298e-05, 2.180718001909554e-05, 2.0526767912087962e-05, 1.384220740874298e-05, 1.3043591025052592e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.632263923554092e-06, "first_to_last_ratio": 0.3802213248794799, "imbalance_abs_log_ratio": 0.9670017619171267, "last_quartile_mean": 1.218307238559646e-05, "mean": 7.265074810192118e-06, "normalized": [ 1.6008181275970341, 0.5739917816715392, 0.235460636588888, 0.49956890329522624, 0.4370066400923714, 0.3438728691875936, 0.4329980193133239, 0.9771405297973998, 0.9929176905310494, 0.7133552305353121, 0.6552622145802756, 0.7166704391631107, 0.6237781231575423, 0.6649176564292614, 0.7618691062673217, 0.8379753775300484, 0.5325892299976221, 0.6974715354559842, 0.6405465790924313, 0.531438133277865, 0.87605151631772, 1.2349821890311432, 1.4485483745005174, 1.55527350554067, 1.5475592171159964, 1.891436311373116, 2.0687037039687666, 1.6168445987295306, 1.977149264052817, 1.9131038449622708, 1.2696779592298815, 1.1310206916163696 ], "peak_layer": 27, "peak_normalized": 2.0687037039687666, "population_cv": 0.5243879132228086, "rest_mean": 6.817757480348519e-06, "spike_contrast": 1.4199080004313516, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.680588391347555e-06, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 1.1630063454504125e-05, 4.170093234279193e-06, 1.7106391396737308e-06, 3.6294054552854504e-06, 3.1748859328217804e-06, 2.4982621198432753e-06, 3.1457630029763095e-06, 7.098999049048871e-06, 7.21362130207126e-06, 5.182579116080888e-06, 4.760529009217862e-06, 5.206664354773238e-06, 4.531794729700778e-06, 4.8306765165762044e-06, 5.535036052606301e-06, 6.0879538068547845e-06, 3.869300599035341e-06, 5.067182883067289e-06, 4.653618816519156e-06, 3.860937795252539e-06, 6.364579803630477e-06, 8.972237992566079e-06, 1.0523812306928448e-05, 1.1299178368062712e-05, 1.1243133485550061e-05, 1.3741426300839521e-05, 1.502928716945462e-05, 1.1746496966225095e-05, 1.4364137314260006e-05, 1.3898842553317081e-05, 9.224305358657148e-06, 8.216949936468154e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.248317332487204e-05, "first_to_last_ratio": 0.36732053671673337, "imbalance_abs_log_ratio": 1.001520415068512, "last_quartile_mean": 6.120859325164929e-05, "mean": 3.520552513691655e-05, "normalized": [ 1.6113007958889953, 0.5636281288577828, 0.3099745506097867, 0.6128259302643165, 0.3776724202532394, 0.30228081296349946, 0.41660627301599373, 0.9147212761628488, 0.9143135144743936, 0.6818048236562128, 0.6378236024718279, 0.669448797145322, 0.5777616284561341, 0.6047361595032282, 0.735292441295308, 0.776547208280745, 0.5187662829625476, 0.6862775953042356, 0.6442540642333291, 0.503974019865995, 0.8786895478652295, 1.1802818108868638, 1.3947424485633424, 1.5774151382314772, 1.403240136016062, 1.8387857304360302, 2.291845133717591, 1.6909973322209617, 2.0609348356508104, 1.9921347125886224, 1.3276799111046451, 1.3032429370526224 ], "peak_layer": 27, "peak_normalized": 2.291845133717591, "population_cv": 0.5527267740898714, "rest_mean": 3.3335239331117247e-05, "spike_contrast": 1.3590743428664134, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.530506848823279e-05, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 5.672669067280367e-05, 1.9842824258375913e-05, 1.0912816833297256e-05, 2.1574858692474663e-05, 1.329615588474553e-05, 1.064195475919405e-05, 1.4666842616861686e-05, 3.220324288122356e-05, 3.2188887416850775e-05, 2.4003296857699752e-05, 2.2454914869740605e-05, 2.356829645577818e-05, 2.0340401533758268e-05, 2.1290054064593278e-05, 2.5886356525006704e-05, 2.733875226113014e-05, 1.826343941502273e-05, 2.416076313238591e-05, 2.2681302652927116e-05, 1.7742670024745166e-05, 3.093472696491517e-05, 4.155244096182287e-05, 4.910264033242129e-05, 5.553372830036096e-05, 4.940180588164367e-05, 6.473541725426912e-05, 8.068561146501452e-05, 5.953244908596389e-05, 7.255629316205159e-05, 7.013414870016277e-05, 4.6741668484173715e-05, 4.588135197991505e-05 ] } } } } } }, "step": 8000 } ], "environment": { "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", "compile": false, "compute_capability": [ 12, 0 ], "cublas_workspace_config": ":4096:8", "cuda": "12.8", "deterministic_algorithms": true, "gpu": "NVIDIA GeForce RTX 5090", "python": "3.10.14", "torch": "2.11.0+cu128" }, "evaluations": [ { "bits_per_byte": 7.9557801867581395, "cross_entropy_nats": 5.514526605606079, "step": 0 }, { "bits_per_byte": 5.165135820292823, "cross_entropy_nats": 3.5801993310451508, "step": 100 }, { "bits_per_byte": 3.109974443324717, "cross_entropy_nats": 2.155670017004013, "step": 500 }, { "bits_per_byte": 1.9592225965086283, "cross_entropy_nats": 1.358029618859291, "step": 2000 }, { "bits_per_byte": 1.8027438806514628, "cross_entropy_nats": 1.249566838145256, "step": 4000 }, { "bits_per_byte": 1.7123525940760473, "cross_entropy_nats": 1.1869123727083206, "step": 8000 } ], "hashes": { "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", "initial_public_parameter_elements": 18985152, "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", "initial_public_parameter_tensors": 227, "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" }, "manifest": { "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", "input_gate_tensor_hashes": { "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" }, "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", "parent_path": "experiments/k3/attnres_gradient/manifest.json", "path": "experiments/k3/attnres_spike/manifest.json", "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" }, "model": { "aggregation_groups": 8, "blocks_per_group": 4, "d_ff": 768, "d_model": 192, "heads": 6, "layers": 32, "parameters": { "core": 18985152, "embedding": 98304, "mixer": 24960, "total": 19010112 } }, "optimizer": { "betas": [ 0.9, 0.95 ], "epsilon": 1e-08, "grad_clip": 1.0, "min_lr": 3e-05, "name": "AdamW", "peak_lr": 0.0003, "warmup_steps": 400, "weight_decay_ndim_ge_2": 0.1 }, "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", "round05_equivalence": { "checks": { "evaluations": true, "final_model_state": true, "final_optimizer_state": true, "post_mlp_element_rms_all_steps": true, "training_history": true }, "passed": true }, "run_kind": "formal", "schema_version": 1, "seed": 2026073001, "steps": 8000, "target_bytes_seen": 65536000, "timing": { "mean_ms": 224.20719212223366, "measured_steps": 7980, "median_ms": 227.38306851533707, "p95_ms": 230.37451171875, "peak_allocated_bytes": 12841009152, "peak_reserved_bytes": 13922992128, "warmup_steps_excluded": 20 }, "training_history": [ { "bits_per_byte": 7.958777844362678, "learning_rate": 7.499999999999999e-07, "loss_nats": 5.516604423522949, "step": 1, "unclipped_grad_norm": 21.323171615600586 }, { "bits_per_byte": 7.247870358756172, "learning_rate": 7.499999999999999e-06, "loss_nats": 5.02384090423584, "step": 10, "unclipped_grad_norm": 11.153014183044434 }, { "bits_per_byte": 6.7336346654397365, "learning_rate": 1.4999999999999999e-05, "loss_nats": 4.667399883270264, "step": 20, "unclipped_grad_norm": 3.4888553619384766 }, { "bits_per_byte": 6.611855126224988, "learning_rate": 2.2499999999999998e-05, "loss_nats": 4.582988739013672, "step": 30, "unclipped_grad_norm": 3.075686454772949 }, { "bits_per_byte": 6.3915955694037825, "learning_rate": 2.9999999999999997e-05, "loss_nats": 4.43031644821167, "step": 40, "unclipped_grad_norm": 3.2318925857543945 }, { "bits_per_byte": 6.093941755902502, "learning_rate": 3.75e-05, "loss_nats": 4.223998546600342, "step": 50, "unclipped_grad_norm": 2.8704335689544678 }, { "bits_per_byte": 5.8858000966610895, "learning_rate": 4.4999999999999996e-05, "loss_nats": 4.079725742340088, "step": 60, "unclipped_grad_norm": 2.669306755065918 }, { "bits_per_byte": 5.715495996470702, "learning_rate": 5.2499999999999995e-05, "loss_nats": 3.9616799354553223, "step": 70, "unclipped_grad_norm": 3.1737496852874756 }, { "bits_per_byte": 5.565897919513593, "learning_rate": 5.9999999999999995e-05, "loss_nats": 3.8579864501953125, "step": 80, "unclipped_grad_norm": 2.5544967651367188 }, { "bits_per_byte": 5.354669171689194, "learning_rate": 6.749999999999999e-05, "loss_nats": 3.711573839187622, "step": 90, "unclipped_grad_norm": 2.425950050354004 }, { "bits_per_byte": 5.180876231443403, "learning_rate": 7.5e-05, "loss_nats": 3.5911097526550293, "step": 100, "unclipped_grad_norm": 2.2783501148223877 }, { "bits_per_byte": 4.992272489270609, "learning_rate": 8.249999999999999e-05, "loss_nats": 3.4603796005249023, "step": 110, "unclipped_grad_norm": 2.1105899810791016 }, { "bits_per_byte": 4.775783480086919, "learning_rate": 8.999999999999999e-05, "loss_nats": 3.3103208541870117, "step": 120, "unclipped_grad_norm": 1.8969556093215942 }, { "bits_per_byte": 4.582375918510225, "learning_rate": 9.75e-05, "loss_nats": 3.1762609481811523, "step": 130, "unclipped_grad_norm": 1.7015109062194824 }, { "bits_per_byte": 4.427903165297061, "learning_rate": 0.00010499999999999999, "loss_nats": 3.0691885948181152, "step": 140, "unclipped_grad_norm": 1.446854829788208 }, { "bits_per_byte": 4.271120341117507, "learning_rate": 0.0001125, "loss_nats": 2.960515022277832, "step": 150, "unclipped_grad_norm": 1.2307006120681763 }, { "bits_per_byte": 4.166711049639903, "learning_rate": 0.00011999999999999999, "loss_nats": 2.888144016265869, "step": 160, "unclipped_grad_norm": 1.0598198175430298 }, { "bits_per_byte": 4.026318516052877, "learning_rate": 0.00012749999999999998, "loss_nats": 2.7908313274383545, "step": 170, "unclipped_grad_norm": 0.9654788374900818 }, { "bits_per_byte": 3.94755837315965, "learning_rate": 0.00013499999999999997, "loss_nats": 2.736238956451416, "step": 180, "unclipped_grad_norm": 0.8419675230979919 }, { "bits_per_byte": 3.838975750676209, "learning_rate": 0.0001425, "loss_nats": 2.660975217819214, "step": 190, "unclipped_grad_norm": 0.817121684551239 }, { "bits_per_byte": 3.774983758012572, "learning_rate": 0.00015, "loss_nats": 2.616619348526001, "step": 200, "unclipped_grad_norm": 0.955824613571167 }, { "bits_per_byte": 3.7344016998190783, "learning_rate": 0.0001575, "loss_nats": 2.5884900093078613, "step": 210, "unclipped_grad_norm": 0.8557147979736328 }, { "bits_per_byte": 3.63166201591605, "learning_rate": 0.00016499999999999997, "loss_nats": 2.5172762870788574, "step": 220, "unclipped_grad_norm": 0.6891153454780579 }, { "bits_per_byte": 3.6208855830129867, "learning_rate": 0.0001725, "loss_nats": 2.5098066329956055, "step": 230, "unclipped_grad_norm": 1.036860466003418 }, { "bits_per_byte": 3.4987607026353063, "learning_rate": 0.00017999999999999998, "loss_nats": 2.4251561164855957, "step": 240, "unclipped_grad_norm": 0.6406477689743042 }, { "bits_per_byte": 3.5387748741156506, "learning_rate": 0.0001875, "loss_nats": 2.4528918266296387, "step": 250, "unclipped_grad_norm": 1.106186866760254 }, { "bits_per_byte": 3.5260464381252943, "learning_rate": 0.000195, "loss_nats": 2.4440691471099854, "step": 260, "unclipped_grad_norm": 0.8556521534919739 }, { "bits_per_byte": 3.5219934949750615, "learning_rate": 0.00020249999999999996, "loss_nats": 2.4412598609924316, "step": 270, "unclipped_grad_norm": 0.9917709827423096 }, { "bits_per_byte": 3.474740573719901, "learning_rate": 0.00020999999999999998, "loss_nats": 2.4085066318511963, "step": 280, "unclipped_grad_norm": 1.2901222705841064 }, { "bits_per_byte": 3.363479149466621, "learning_rate": 0.00021749999999999997, "loss_nats": 2.331386089324951, "step": 290, "unclipped_grad_norm": 0.7895627617835999 }, { "bits_per_byte": 3.4037392561376993, "learning_rate": 0.000225, "loss_nats": 2.3592922687530518, "step": 300, "unclipped_grad_norm": 1.5281883478164673 }, { "bits_per_byte": 3.3541442751430894, "learning_rate": 0.00023249999999999996, "loss_nats": 2.324915647506714, "step": 310, "unclipped_grad_norm": 0.8648361563682556 }, { "bits_per_byte": 3.3434108379027414, "learning_rate": 0.00023999999999999998, "loss_nats": 2.3174757957458496, "step": 320, "unclipped_grad_norm": 1.4613573551177979 }, { "bits_per_byte": 3.3325662998699364, "learning_rate": 0.0002475, "loss_nats": 2.3099589347839355, "step": 330, "unclipped_grad_norm": 1.1215400695800781 }, { "bits_per_byte": 3.4180712583647006, "learning_rate": 0.00025499999999999996, "loss_nats": 2.3692264556884766, "step": 340, "unclipped_grad_norm": 1.1166281700134277 }, { "bits_per_byte": 3.3577222022116393, "learning_rate": 0.0002625, "loss_nats": 2.3273956775665283, "step": 350, "unclipped_grad_norm": 1.0810235738754272 }, { "bits_per_byte": 3.2923828974611427, "learning_rate": 0.00026999999999999995, "loss_nats": 2.2821059226989746, "step": 360, "unclipped_grad_norm": 1.128420114517212 }, { "bits_per_byte": 3.291989401155969, "learning_rate": 0.00027749999999999997, "loss_nats": 2.2818331718444824, "step": 370, "unclipped_grad_norm": 1.0846341848373413 }, { "bits_per_byte": 3.3389695579268603, "learning_rate": 0.000285, "loss_nats": 2.3143973350524902, "step": 380, "unclipped_grad_norm": 1.0868512392044067 }, { "bits_per_byte": 3.256548592327368, "learning_rate": 0.0002925, "loss_nats": 2.257267475128174, "step": 390, "unclipped_grad_norm": 1.1476690769195557 }, { "bits_per_byte": 3.2058776878689605, "learning_rate": 0.0003, "loss_nats": 2.2221450805664062, "step": 400, "unclipped_grad_norm": 1.1263198852539062 }, { "bits_per_byte": 3.195324144481415, "learning_rate": 0.0002999988466112388, "loss_nats": 2.214829921722412, "step": 410, "unclipped_grad_norm": 1.1377208232879639 }, { "bits_per_byte": 3.227808915506986, "learning_rate": 0.00029999538646466355, "loss_nats": 2.237346649169922, "step": 420, "unclipped_grad_norm": 1.2929860353469849 }, { "bits_per_byte": 3.1876103786249166, "learning_rate": 0.0002999896196193985, "loss_nats": 2.2094831466674805, "step": 430, "unclipped_grad_norm": 1.071694016456604 }, { "bits_per_byte": 3.2374495749837515, "learning_rate": 0.00029998154617398326, "loss_nats": 2.2440290451049805, "step": 440, "unclipped_grad_norm": 0.9089367985725403 }, { "bits_per_byte": 3.155426921203657, "learning_rate": 0.0002999711662663707, "loss_nats": 2.1871752738952637, "step": 450, "unclipped_grad_norm": 1.1129212379455566 }, { "bits_per_byte": 3.188086770567806, "learning_rate": 0.0002999584800739247, "loss_nats": 2.209813356399536, "step": 460, "unclipped_grad_norm": 1.3905333280563354 }, { "bits_per_byte": 3.128408790718412, "learning_rate": 0.0002999434878134173, "loss_nats": 2.168447732925415, "step": 470, "unclipped_grad_norm": 1.059533953666687 }, { "bits_per_byte": 3.136691131218663, "learning_rate": 0.000299926189741025, "loss_nats": 2.1741886138916016, "step": 480, "unclipped_grad_norm": 1.0161854028701782 }, { "bits_per_byte": 3.0887341129908705, "learning_rate": 0.000299906586152324, "loss_nats": 2.1409473419189453, "step": 490, "unclipped_grad_norm": 1.385905385017395 }, { "bits_per_byte": 3.033835874974255, "learning_rate": 0.0002998846773822856, "loss_nats": 2.1028947830200195, "step": 500, "unclipped_grad_norm": 1.1716443300247192 }, { "bits_per_byte": 3.0607071322756636, "learning_rate": 0.0002998604638052702, "loss_nats": 2.121520519256592, "step": 510, "unclipped_grad_norm": 0.947066605091095 }, { "bits_per_byte": 2.997100036784657, "learning_rate": 0.0002998339458350211, "loss_nats": 2.0774314403533936, "step": 520, "unclipped_grad_norm": 1.352820634841919 }, { "bits_per_byte": 3.0643050093317137, "learning_rate": 0.0002998051239246572, "loss_nats": 2.124014377593994, "step": 530, "unclipped_grad_norm": 1.2646857500076294 }, { "bits_per_byte": 3.008989197438803, "learning_rate": 0.00029977399856666537, "loss_nats": 2.085672378540039, "step": 540, "unclipped_grad_norm": 1.0451104640960693 }, { "bits_per_byte": 2.978105584892765, "learning_rate": 0.0002997405702928922, "loss_nats": 2.064265489578247, "step": 550, "unclipped_grad_norm": 1.2619587182998657 }, { "bits_per_byte": 2.968300166036473, "learning_rate": 0.0002997048396745345, "loss_nats": 2.057468891143799, "step": 560, "unclipped_grad_norm": 0.9861598610877991 }, { "bits_per_byte": 2.9221661639078262, "learning_rate": 0.0002996668073221302, "loss_nats": 2.025491237640381, "step": 570, "unclipped_grad_norm": 1.1261875629425049 }, { "bits_per_byte": 2.96999178739036, "learning_rate": 0.000299626473885547, "loss_nats": 2.0586414337158203, "step": 580, "unclipped_grad_norm": 0.8640331625938416 }, { "bits_per_byte": 2.883698804389342, "learning_rate": 0.00029958384005397226, "loss_nats": 1.9988276958465576, "step": 590, "unclipped_grad_norm": 1.0034456253051758 }, { "bits_per_byte": 2.9404706214044567, "learning_rate": 0.0002995389065559004, "loss_nats": 2.0381789207458496, "step": 600, "unclipped_grad_norm": 0.8714175224304199 }, { "bits_per_byte": 2.8380133330141457, "learning_rate": 0.000299491674159121, "loss_nats": 1.967160940170288, "step": 610, "unclipped_grad_norm": 0.9427422881126404 }, { "bits_per_byte": 2.894644124255553, "learning_rate": 0.00029944214367070535, "loss_nats": 2.0064144134521484, "step": 620, "unclipped_grad_norm": 1.0238139629364014 }, { "bits_per_byte": 2.81883675149499, "learning_rate": 0.00029939031593699284, "loss_nats": 1.9538687467575073, "step": 630, "unclipped_grad_norm": 0.9875481128692627 }, { "bits_per_byte": 2.7787291934352267, "learning_rate": 0.00029933619184357654, "loss_nats": 1.9260683059692383, "step": 640, "unclipped_grad_norm": 1.0125378370285034 }, { "bits_per_byte": 2.728423580127252, "learning_rate": 0.00029927977231528774, "loss_nats": 1.8911991119384766, "step": 650, "unclipped_grad_norm": 1.5507988929748535 }, { "bits_per_byte": 2.649404087396524, "learning_rate": 0.00029922105831618057, "loss_nats": 1.8364269733428955, "step": 660, "unclipped_grad_norm": 0.9040125012397766 }, { "bits_per_byte": 2.7024973674923345, "learning_rate": 0.0002991600508495153, "loss_nats": 1.8732284307479858, "step": 670, "unclipped_grad_norm": 1.205861210823059 }, { "bits_per_byte": 2.641403626461037, "learning_rate": 0.00029909675095774114, "loss_nats": 1.8308814764022827, "step": 680, "unclipped_grad_norm": 1.3370649814605713 }, { "bits_per_byte": 2.7325929555321187, "learning_rate": 0.0002990311597224787, "loss_nats": 1.8940891027450562, "step": 690, "unclipped_grad_norm": 1.6124995946884155 }, { "bits_per_byte": 2.6130706886099455, "learning_rate": 0.00029896327826450103, "loss_nats": 1.8112425804138184, "step": 700, "unclipped_grad_norm": 1.08383309841156 }, { "bits_per_byte": 2.628106443844277, "learning_rate": 0.0002988931077437152, "loss_nats": 1.821664571762085, "step": 710, "unclipped_grad_norm": 1.0767779350280762 }, { "bits_per_byte": 2.6725746220166666, "learning_rate": 0.00029882064935914164, "loss_nats": 1.852487564086914, "step": 720, "unclipped_grad_norm": 1.296352505683899 }, { "bits_per_byte": 2.608218370098502, "learning_rate": 0.0002987459043488943, "loss_nats": 1.8078792095184326, "step": 730, "unclipped_grad_norm": 1.0864766836166382 }, { "bits_per_byte": 2.555699340073923, "learning_rate": 0.0002986688739901591, "loss_nats": 1.7714757919311523, "step": 740, "unclipped_grad_norm": 1.0339913368225098 }, { "bits_per_byte": 2.515936435246489, "learning_rate": 0.0002985895595991723, "loss_nats": 1.743914246559143, "step": 750, "unclipped_grad_norm": 1.1017143726348877 }, { "bits_per_byte": 2.4657125136976736, "learning_rate": 0.00029850796253119824, "loss_nats": 1.709101676940918, "step": 760, "unclipped_grad_norm": 1.7173939943313599 }, { "bits_per_byte": 2.601046349592221, "learning_rate": 0.00029842408418050544, "loss_nats": 1.802907943725586, "step": 770, "unclipped_grad_norm": 0.9487742781639099 }, { "bits_per_byte": 2.4794518637098015, "learning_rate": 0.0002983379259803436, "loss_nats": 1.7186250686645508, "step": 780, "unclipped_grad_norm": 0.9247332811355591 }, { "bits_per_byte": 2.466098098800908, "learning_rate": 0.0002982494894029186, "loss_nats": 1.7093689441680908, "step": 790, "unclipped_grad_norm": 1.9121034145355225 }, { "bits_per_byte": 2.4025570446478333, "learning_rate": 0.0002981587759593675, "loss_nats": 1.66532564163208, "step": 800, "unclipped_grad_norm": 1.1767630577087402 }, { "bits_per_byte": 2.455517898102479, "learning_rate": 0.00029806578719973283, "loss_nats": 1.7020353078842163, "step": 810, "unclipped_grad_norm": 1.1299240589141846 }, { "bits_per_byte": 2.400107839716896, "learning_rate": 0.00029797052471293596, "loss_nats": 1.6636279821395874, "step": 820, "unclipped_grad_norm": 1.0006046295166016 }, { "bits_per_byte": 2.4500607166079513, "learning_rate": 0.0002978729901267499, "loss_nats": 1.6982526779174805, "step": 830, "unclipped_grad_norm": 1.0554603338241577 }, { "bits_per_byte": 2.5153210813217015, "learning_rate": 0.0002977731851077716, "loss_nats": 1.7434877157211304, "step": 840, "unclipped_grad_norm": 1.0736713409423828 }, { "bits_per_byte": 2.399556119372928, "learning_rate": 0.0002976711113613935, "loss_nats": 1.6632455587387085, "step": 850, "unclipped_grad_norm": 0.8279580473899841 }, { "bits_per_byte": 2.414819407654351, "learning_rate": 0.0002975667706317743, "loss_nats": 1.6738252639770508, "step": 860, "unclipped_grad_norm": 1.2342168092727661 }, { "bits_per_byte": 2.3222158652452247, "learning_rate": 0.00029746016470180915, "loss_nats": 1.6096373796463013, "step": 870, "unclipped_grad_norm": 1.0648512840270996 }, { "bits_per_byte": 2.3629336177502487, "learning_rate": 0.00029735129539309926, "loss_nats": 1.6378607749938965, "step": 880, "unclipped_grad_norm": 0.9258757829666138 }, { "bits_per_byte": 2.2961121665146167, "learning_rate": 0.0002972401645659207, "loss_nats": 1.5915436744689941, "step": 890, "unclipped_grad_norm": 1.0717874765396118 }, { "bits_per_byte": 2.3158979105831285, "learning_rate": 0.00029712677411919283, "loss_nats": 1.6052581071853638, "step": 900, "unclipped_grad_norm": 0.9598665833473206 }, { "bits_per_byte": 2.3301258633063577, "learning_rate": 0.0002970111259904454, "loss_nats": 1.6151201725006104, "step": 910, "unclipped_grad_norm": 1.0995051860809326 }, { "bits_per_byte": 2.3123376977276173, "learning_rate": 0.0002968932221557859, "loss_nats": 1.602790355682373, "step": 920, "unclipped_grad_norm": 1.1837356090545654 }, { "bits_per_byte": 2.3340250539667196, "learning_rate": 0.0002967730646298655, "loss_nats": 1.6178228855133057, "step": 930, "unclipped_grad_norm": 1.1502224206924438 }, { "bits_per_byte": 2.330418921743428, "learning_rate": 0.0002966506554658449, "loss_nats": 1.6153233051300049, "step": 940, "unclipped_grad_norm": 1.0088714361190796 }, { "bits_per_byte": 2.2995397807463123, "learning_rate": 0.00029652599675535876, "loss_nats": 1.5939195156097412, "step": 950, "unclipped_grad_norm": 0.9483612179756165 }, { "bits_per_byte": 2.3536873144919115, "learning_rate": 0.0002963990906284808, "loss_nats": 1.6314517259597778, "step": 960, "unclipped_grad_norm": 0.9582105278968811 }, { "bits_per_byte": 2.2526236015215417, "learning_rate": 0.0002962699392536863, "loss_nats": 1.5613996982574463, "step": 970, "unclipped_grad_norm": 1.2002545595169067 }, { "bits_per_byte": 2.3230659754884395, "learning_rate": 0.0002961385448378161, "loss_nats": 1.6102266311645508, "step": 980, "unclipped_grad_norm": 0.9191399812698364 }, { "bits_per_byte": 2.285470911975131, "learning_rate": 0.00029600490962603817, "loss_nats": 1.584167718887329, "step": 990, "unclipped_grad_norm": 1.053741216659546 }, { "bits_per_byte": 2.248061417742104, "learning_rate": 0.00029586903590180956, "loss_nats": 1.5582374334335327, "step": 1000, "unclipped_grad_norm": 0.9024814963340759 }, { "bits_per_byte": 2.2402749032242903, "learning_rate": 0.00029573092598683735, "loss_nats": 1.552840232849121, "step": 1010, "unclipped_grad_norm": 1.048907995223999 }, { "bits_per_byte": 2.2041726491204714, "learning_rate": 0.00029559058224103895, "loss_nats": 1.5278160572052002, "step": 1020, "unclipped_grad_norm": 1.0486118793487549 }, { "bits_per_byte": 2.216881995036582, "learning_rate": 0.0002954480070625017, "loss_nats": 1.5366255044937134, "step": 1030, "unclipped_grad_norm": 0.9243784546852112 }, { "bits_per_byte": 2.2610579746851553, "learning_rate": 0.00029530320288744205, "loss_nats": 1.5672459602355957, "step": 1040, "unclipped_grad_norm": 1.102870225906372 }, { "bits_per_byte": 2.2528270569975124, "learning_rate": 0.0002951561721901638, "loss_nats": 1.5615407228469849, "step": 1050, "unclipped_grad_norm": 1.0764544010162354 }, { "bits_per_byte": 2.14889054582713, "learning_rate": 0.00029500691748301594, "loss_nats": 1.489497423171997, "step": 1060, "unclipped_grad_norm": 0.971049427986145 }, { "bits_per_byte": 2.2350196293618754, "learning_rate": 0.00029485544131634963, "loss_nats": 1.5491975545883179, "step": 1070, "unclipped_grad_norm": 1.0203380584716797 }, { "bits_per_byte": 2.217042282867186, "learning_rate": 0.0002947017462784745, "loss_nats": 1.5367366075515747, "step": 1080, "unclipped_grad_norm": 1.023221492767334 }, { "bits_per_byte": 2.186512266651322, "learning_rate": 0.00029454583499561476, "loss_nats": 1.5155748128890991, "step": 1090, "unclipped_grad_norm": 1.111006498336792 }, { "bits_per_byte": 2.2323391077655277, "learning_rate": 0.00029438771013186413, "loss_nats": 1.5473395586013794, "step": 1100, "unclipped_grad_norm": 0.9547863602638245 }, { "bits_per_byte": 2.116467344590568, "learning_rate": 0.0002942273743891401, "loss_nats": 1.4670233726501465, "step": 1110, "unclipped_grad_norm": 1.1085364818572998 }, { "bits_per_byte": 2.143238164023873, "learning_rate": 0.00029406483050713824, "loss_nats": 1.485579490661621, "step": 1120, "unclipped_grad_norm": 0.901618242263794 }, { "bits_per_byte": 2.077939275196329, "learning_rate": 0.00029390008126328504, "loss_nats": 1.4403177499771118, "step": 1130, "unclipped_grad_norm": 0.8683149814605713 }, { "bits_per_byte": 2.224320932617002, "learning_rate": 0.00029373312947269045, "loss_nats": 1.5417817831039429, "step": 1140, "unclipped_grad_norm": 1.030465006828308 }, { "bits_per_byte": 2.0642131678483175, "learning_rate": 0.00029356397798809995, "loss_nats": 1.4308035373687744, "step": 1150, "unclipped_grad_norm": 0.827274739742279 }, { "bits_per_byte": 2.103388407957753, "learning_rate": 0.0002933926296998457, "loss_nats": 1.4579577445983887, "step": 1160, "unclipped_grad_norm": 0.9158841967582703 }, { "bits_per_byte": 2.1566975262803965, "learning_rate": 0.0002932190875357972, "loss_nats": 1.4949088096618652, "step": 1170, "unclipped_grad_norm": 0.9397322535514832 }, { "bits_per_byte": 2.1322636071070153, "learning_rate": 0.00029304335446131126, "loss_nats": 1.4779725074768066, "step": 1180, "unclipped_grad_norm": 0.8510220050811768 }, { "bits_per_byte": 2.068760561119781, "learning_rate": 0.00029286543347918116, "loss_nats": 1.4339555501937866, "step": 1190, "unclipped_grad_norm": 0.9270830154418945 }, { "bits_per_byte": 2.0310528489701576, "learning_rate": 0.0002926853276295856, "loss_nats": 1.4078185558319092, "step": 1200, "unclipped_grad_norm": 0.8815417885780334 }, { "bits_per_byte": 2.1008361854189492, "learning_rate": 0.0002925030399900368, "loss_nats": 1.456188678741455, "step": 1210, "unclipped_grad_norm": 1.2037686109542847 }, { "bits_per_byte": 2.1123913557651197, "learning_rate": 0.00029231857367532726, "loss_nats": 1.464198112487793, "step": 1220, "unclipped_grad_norm": 0.8964646458625793 }, { "bits_per_byte": 2.0711254945517954, "learning_rate": 0.0002921319318374776, "loss_nats": 1.4355947971343994, "step": 1230, "unclipped_grad_norm": 0.8661414980888367 }, { "bits_per_byte": 2.0159510523978943, "learning_rate": 0.00029194311766568174, "loss_nats": 1.397350788116455, "step": 1240, "unclipped_grad_norm": 0.9392534494400024 }, { "bits_per_byte": 2.104896867788471, "learning_rate": 0.000291752134386253, "loss_nats": 1.4590033292770386, "step": 1250, "unclipped_grad_norm": 0.8430818915367126 }, { "bits_per_byte": 2.08180441329182, "learning_rate": 0.00029155898526256856, "loss_nats": 1.442996859550476, "step": 1260, "unclipped_grad_norm": 0.8917548656463623 }, { "bits_per_byte": 2.087352401626003, "learning_rate": 0.0002913636735950141, "loss_nats": 1.4468424320220947, "step": 1270, "unclipped_grad_norm": 0.9866762161254883 }, { "bits_per_byte": 1.9607164163161914, "learning_rate": 0.0002911662027209272, "loss_nats": 1.359065055847168, "step": 1280, "unclipped_grad_norm": 0.8596939444541931 }, { "bits_per_byte": 2.088993460080536, "learning_rate": 0.00029096657601454013, "loss_nats": 1.4479799270629883, "step": 1290, "unclipped_grad_norm": 1.007126808166504 }, { "bits_per_byte": 2.034349412421896, "learning_rate": 0.00029076479688692266, "loss_nats": 1.4101035594940186, "step": 1300, "unclipped_grad_norm": 0.8474438786506653 }, { "bits_per_byte": 2.0247273270714237, "learning_rate": 0.00029056086878592323, "loss_nats": 1.4034340381622314, "step": 1310, "unclipped_grad_norm": 0.8226730823516846 }, { "bits_per_byte": 2.020511000402871, "learning_rate": 0.00029035479519611045, "loss_nats": 1.4005115032196045, "step": 1320, "unclipped_grad_norm": 0.9414357542991638 }, { "bits_per_byte": 2.085571349293668, "learning_rate": 0.0002901465796387134, "loss_nats": 1.4456079006195068, "step": 1330, "unclipped_grad_norm": 0.9917200207710266 }, { "bits_per_byte": 2.0422444479924042, "learning_rate": 0.00028993622567156154, "loss_nats": 1.4155759811401367, "step": 1340, "unclipped_grad_norm": 0.8823419213294983 }, { "bits_per_byte": 2.0447034559344406, "learning_rate": 0.0002897237368890237, "loss_nats": 1.4172804355621338, "step": 1350, "unclipped_grad_norm": 0.8861486315727234 }, { "bits_per_byte": 2.0988803987133373, "learning_rate": 0.0002895091169219469, "loss_nats": 1.4548330307006836, "step": 1360, "unclipped_grad_norm": 0.9090827107429504 }, { "bits_per_byte": 2.082760980795919, "learning_rate": 0.00028929236943759425, "loss_nats": 1.4436599016189575, "step": 1370, "unclipped_grad_norm": 0.8366561532020569 }, { "bits_per_byte": 2.0317923743688677, "learning_rate": 0.00028907349813958225, "loss_nats": 1.4083311557769775, "step": 1380, "unclipped_grad_norm": 0.8379060626029968 }, { "bits_per_byte": 2.0432778917414387, "learning_rate": 0.0002888525067678175, "loss_nats": 1.4162923097610474, "step": 1390, "unclipped_grad_norm": 0.8665340542793274 }, { "bits_per_byte": 1.9746810636009497, "learning_rate": 0.00028862939909843273, "loss_nats": 1.3687446117401123, "step": 1400, "unclipped_grad_norm": 0.8112426996231079 }, { "bits_per_byte": 2.0034977886489185, "learning_rate": 0.0002884041789437225, "loss_nats": 1.388718843460083, "step": 1410, "unclipped_grad_norm": 0.8745980262756348 }, { "bits_per_byte": 2.019574210903621, "learning_rate": 0.0002881768501520778, "loss_nats": 1.3998621702194214, "step": 1420, "unclipped_grad_norm": 0.8596643209457397 }, { "bits_per_byte": 2.0775151659793014, "learning_rate": 0.0002879474166079205, "loss_nats": 1.4400237798690796, "step": 1430, "unclipped_grad_norm": 0.913244903087616 }, { "bits_per_byte": 2.027370700415185, "learning_rate": 0.00028771588223163694, "loss_nats": 1.405266284942627, "step": 1440, "unclipped_grad_norm": 1.0067864656448364 }, { "bits_per_byte": 1.9631205617926026, "learning_rate": 0.00028748225097951054, "loss_nats": 1.3607314825057983, "step": 1450, "unclipped_grad_norm": 0.8125852942466736 }, { "bits_per_byte": 1.9959419028659118, "learning_rate": 0.000287246526843655, "loss_nats": 1.383481502532959, "step": 1460, "unclipped_grad_norm": 0.9857028126716614 }, { "bits_per_byte": 1.9364156117145783, "learning_rate": 0.00028700871385194533, "loss_nats": 1.3422210216522217, "step": 1470, "unclipped_grad_norm": 0.8698804974555969 }, { "bits_per_byte": 1.996789089404062, "learning_rate": 0.00028676881606794945, "loss_nats": 1.3840687274932861, "step": 1480, "unclipped_grad_norm": 0.8397032022476196 }, { "bits_per_byte": 2.017818268038312, "learning_rate": 0.00028652683759085854, "loss_nats": 1.398645043373108, "step": 1490, "unclipped_grad_norm": 0.8181726932525635 }, { "bits_per_byte": 1.996446843928845, "learning_rate": 0.0002862827825554171, "loss_nats": 1.38383150100708, "step": 1500, "unclipped_grad_norm": 0.8242432475090027 }, { "bits_per_byte": 2.0162594172908914, "learning_rate": 0.00028603665513185245, "loss_nats": 1.3975645303726196, "step": 1510, "unclipped_grad_norm": 0.9299814701080322 }, { "bits_per_byte": 1.980743108078562, "learning_rate": 0.000285788459525803, "loss_nats": 1.3729465007781982, "step": 1520, "unclipped_grad_norm": 0.8446369171142578 }, { "bits_per_byte": 2.047233664693934, "learning_rate": 0.00028553819997824707, "loss_nats": 1.4190342426300049, "step": 1530, "unclipped_grad_norm": 0.9717140197753906 }, { "bits_per_byte": 1.993557363411699, "learning_rate": 0.00028528588076542964, "loss_nats": 1.3818286657333374, "step": 1540, "unclipped_grad_norm": 0.8417568802833557 }, { "bits_per_byte": 2.049996737962696, "learning_rate": 0.00028503150619878996, "loss_nats": 1.4209494590759277, "step": 1550, "unclipped_grad_norm": 0.8546327352523804 }, { "bits_per_byte": 1.966114779744124, "learning_rate": 0.0002847750806248876, "loss_nats": 1.3628069162368774, "step": 1560, "unclipped_grad_norm": 0.9258830547332764 }, { "bits_per_byte": 2.0427641795633114, "learning_rate": 0.000284516608425328, "loss_nats": 1.4159362316131592, "step": 1570, "unclipped_grad_norm": 0.8567822575569153 }, { "bits_per_byte": 1.9830621221427962, "learning_rate": 0.000284256094016688, "loss_nats": 1.374553918838501, "step": 1580, "unclipped_grad_norm": 0.8522606492042542 }, { "bits_per_byte": 1.9876789964052082, "learning_rate": 0.0002839935418504399, "loss_nats": 1.3777540922164917, "step": 1590, "unclipped_grad_norm": 0.8731296062469482 }, { "bits_per_byte": 2.0122188568915207, "learning_rate": 0.000283728956412876, "loss_nats": 1.3947638273239136, "step": 1600, "unclipped_grad_norm": 0.8627029657363892 }, { "bits_per_byte": 1.9498011934138815, "learning_rate": 0.00028346234222503136, "loss_nats": 1.3514991998672485, "step": 1610, "unclipped_grad_norm": 0.7740172147750854 }, { "bits_per_byte": 1.9095543294069197, "learning_rate": 0.00028319370384260675, "loss_nats": 1.3236021995544434, "step": 1620, "unclipped_grad_norm": 0.7910023331642151 }, { "bits_per_byte": 1.9407452749500496, "learning_rate": 0.00028292304585589105, "loss_nats": 1.3452221155166626, "step": 1630, "unclipped_grad_norm": 0.8484107851982117 }, { "bits_per_byte": 1.9911346438089947, "learning_rate": 0.0002826503728896824, "loss_nats": 1.3801493644714355, "step": 1640, "unclipped_grad_norm": 0.8722788691520691 }, { "bits_per_byte": 2.0368601871418424, "learning_rate": 0.0002823756896032095, "loss_nats": 1.4118438959121704, "step": 1650, "unclipped_grad_norm": 0.8482462763786316 }, { "bits_per_byte": 1.9665676100638458, "learning_rate": 0.0002820990006900518, "loss_nats": 1.3631207942962646, "step": 1660, "unclipped_grad_norm": 0.8444085121154785 }, { "bits_per_byte": 1.9945896032821775, "learning_rate": 0.00028182031087805947, "loss_nats": 1.3825441598892212, "step": 1670, "unclipped_grad_norm": 0.8560684323310852 }, { "bits_per_byte": 2.0061820938635853, "learning_rate": 0.0002815396249292725, "loss_nats": 1.3905794620513916, "step": 1680, "unclipped_grad_norm": 0.8526912927627563 }, { "bits_per_byte": 2.0514603103215387, "learning_rate": 0.0002812569476398391, "loss_nats": 1.4219639301300049, "step": 1690, "unclipped_grad_norm": 0.8917502760887146 }, { "bits_per_byte": 1.9422809080396035, "learning_rate": 0.00028097228383993446, "loss_nats": 1.3462865352630615, "step": 1700, "unclipped_grad_norm": 0.7474871873855591 }, { "bits_per_byte": 1.878015978909047, "learning_rate": 0.0002806856383936774, "loss_nats": 1.3017414808273315, "step": 1710, "unclipped_grad_norm": 0.8072315454483032 }, { "bits_per_byte": 1.8905051870320533, "learning_rate": 0.00028039701619904774, "loss_nats": 1.3103983402252197, "step": 1720, "unclipped_grad_norm": 0.845556914806366 }, { "bits_per_byte": 1.8762399140735866, "learning_rate": 0.0002801064221878024, "loss_nats": 1.3005104064941406, "step": 1730, "unclipped_grad_norm": 0.9344940185546875 }, { "bits_per_byte": 1.9632232354351677, "learning_rate": 0.00027981386132539134, "loss_nats": 1.3608026504516602, "step": 1740, "unclipped_grad_norm": 0.7860697507858276 }, { "bits_per_byte": 1.989613629244764, "learning_rate": 0.0002795193386108724, "loss_nats": 1.3790950775146484, "step": 1750, "unclipped_grad_norm": 0.8348115086555481 }, { "bits_per_byte": 1.9869124697303127, "learning_rate": 0.00027922285907682625, "loss_nats": 1.3772227764129639, "step": 1760, "unclipped_grad_norm": 0.8862095475196838 }, { "bits_per_byte": 1.9928130224987648, "learning_rate": 0.00027892442778927005, "loss_nats": 1.3813127279281616, "step": 1770, "unclipped_grad_norm": 0.816246509552002 }, { "bits_per_byte": 1.8514405317671223, "learning_rate": 0.00027862404984757116, "loss_nats": 1.2833207845687866, "step": 1780, "unclipped_grad_norm": 0.8118871450424194 }, { "bits_per_byte": 1.9588221993995885, "learning_rate": 0.0002783217303843597, "loss_nats": 1.3577520847320557, "step": 1790, "unclipped_grad_norm": 0.839783251285553 }, { "bits_per_byte": 1.8827697513615458, "learning_rate": 0.00027801747456544134, "loss_nats": 1.3050365447998047, "step": 1800, "unclipped_grad_norm": 0.8949951529502869 }, { "bits_per_byte": 1.92937567388416, "learning_rate": 0.00027771128758970843, "loss_nats": 1.33734130859375, "step": 1810, "unclipped_grad_norm": 0.7295374274253845 }, { "bits_per_byte": 1.8974312723476003, "learning_rate": 0.00027740317468905165, "loss_nats": 1.3151991367340088, "step": 1820, "unclipped_grad_norm": 0.7755441069602966 }, { "bits_per_byte": 1.8886371114783813, "learning_rate": 0.0002770931411282704, "loss_nats": 1.3091034889221191, "step": 1830, "unclipped_grad_norm": 0.7666956186294556 }, { "bits_per_byte": 1.8876932706904463, "learning_rate": 0.0002767811922049827, "loss_nats": 1.3084492683410645, "step": 1840, "unclipped_grad_norm": 0.8784642219543457 }, { "bits_per_byte": 1.8964055678178544, "learning_rate": 0.00027646733324953503, "loss_nats": 1.314488172531128, "step": 1850, "unclipped_grad_norm": 0.8603965044021606 }, { "bits_per_byte": 1.8927124123732264, "learning_rate": 0.00027615156962491097, "loss_nats": 1.3119282722473145, "step": 1860, "unclipped_grad_norm": 0.8056252002716064 }, { "bits_per_byte": 1.8818489562488263, "learning_rate": 0.0002758339067266396, "loss_nats": 1.3043982982635498, "step": 1870, "unclipped_grad_norm": 0.8151036500930786 }, { "bits_per_byte": 1.8811881989042114, "learning_rate": 0.0002755143499827035, "loss_nats": 1.3039402961730957, "step": 1880, "unclipped_grad_norm": 0.79389888048172 }, { "bits_per_byte": 1.8949089747900463, "learning_rate": 0.0002751929048534456, "loss_nats": 1.313450813293457, "step": 1890, "unclipped_grad_norm": 0.8192400932312012 }, { "bits_per_byte": 1.9233574849825177, "learning_rate": 0.0002748695768314763, "loss_nats": 1.3331698179244995, "step": 1900, "unclipped_grad_norm": 0.7950363159179688 }, { "bits_per_byte": 1.8611783614416253, "learning_rate": 0.00027454437144157947, "loss_nats": 1.2900705337524414, "step": 1910, "unclipped_grad_norm": 0.8037525415420532 }, { "bits_per_byte": 1.855479888287937, "learning_rate": 0.00027421729424061787, "loss_nats": 1.2861206531524658, "step": 1920, "unclipped_grad_norm": 0.7507341504096985 }, { "bits_per_byte": 1.9602173226633874, "learning_rate": 0.0002738883508174385, "loss_nats": 1.3587191104888916, "step": 1930, "unclipped_grad_norm": 0.8329893350601196 }, { "bits_per_byte": 1.8066889261861108, "learning_rate": 0.0002735575467927768, "loss_nats": 1.2523013353347778, "step": 1940, "unclipped_grad_norm": 0.7595474123954773 }, { "bits_per_byte": 1.9033319970987033, "learning_rate": 0.0002732248878191609, "loss_nats": 1.319289207458496, "step": 1950, "unclipped_grad_norm": 0.7690061330795288 }, { "bits_per_byte": 1.816397862775269, "learning_rate": 0.0002728903795808148, "loss_nats": 1.259031057357788, "step": 1960, "unclipped_grad_norm": 0.7758741974830627 }, { "bits_per_byte": 1.909067618504978, "learning_rate": 0.0002725540277935614, "loss_nats": 1.3232648372650146, "step": 1970, "unclipped_grad_norm": 0.7503845691680908 }, { "bits_per_byte": 1.9405849871194454, "learning_rate": 0.00027221583820472456, "loss_nats": 1.3451110124588013, "step": 1980, "unclipped_grad_norm": 0.8655099868774414 }, { "bits_per_byte": 1.9436471382180576, "learning_rate": 0.0002718758165930314, "loss_nats": 1.347233533859253, "step": 1990, "unclipped_grad_norm": 0.8436980247497559 }, { "bits_per_byte": 1.9015801818170146, "learning_rate": 0.0002715339687685131, "loss_nats": 1.3180749416351318, "step": 2000, "unclipped_grad_norm": 0.8021923303604126 }, { "bits_per_byte": 1.8746163978494417, "learning_rate": 0.00027119030057240576, "loss_nats": 1.2993850708007812, "step": 2010, "unclipped_grad_norm": 0.7083144187927246 }, { "bits_per_byte": 1.9072580170525995, "learning_rate": 0.0002708448178770506, "loss_nats": 1.3220105171203613, "step": 2020, "unclipped_grad_norm": 0.7678324580192566 }, { "bits_per_byte": 1.983610230851096, "learning_rate": 0.00027049752658579376, "loss_nats": 1.3749338388442993, "step": 2030, "unclipped_grad_norm": 0.8317387104034424 }, { "bits_per_byte": 1.909348466173837, "learning_rate": 0.00027014843263288514, "loss_nats": 1.323459506034851, "step": 2040, "unclipped_grad_norm": 0.8302199840545654 }, { "bits_per_byte": 1.961244747019642, "learning_rate": 0.00026979754198337724, "loss_nats": 1.359431266784668, "step": 2050, "unclipped_grad_norm": 0.7990598082542419 }, { "bits_per_byte": 1.9220292629699043, "learning_rate": 0.00026944486063302305, "loss_nats": 1.3322491645812988, "step": 2060, "unclipped_grad_norm": 0.8404503464698792 }, { "bits_per_byte": 1.8663830724046866, "learning_rate": 0.0002690903946081738, "loss_nats": 1.2936781644821167, "step": 2070, "unclipped_grad_norm": 0.7720745801925659 }, { "bits_per_byte": 1.8672120287818452, "learning_rate": 0.00026873414996567596, "loss_nats": 1.2942527532577515, "step": 2080, "unclipped_grad_norm": 0.8028935790061951 }, { "bits_per_byte": 1.857879218250124, "learning_rate": 0.00026837613279276736, "loss_nats": 1.2877837419509888, "step": 2090, "unclipped_grad_norm": 0.7713416814804077 }, { "bits_per_byte": 1.8603779541845098, "learning_rate": 0.0002680163492069739, "loss_nats": 1.289515733718872, "step": 2100, "unclipped_grad_norm": 0.7443708181381226 }, { "bits_per_byte": 1.897931741861611, "learning_rate": 0.00026765480535600414, "loss_nats": 1.3155460357666016, "step": 2110, "unclipped_grad_norm": 0.7534405589103699 }, { "bits_per_byte": 1.8710701155886507, "learning_rate": 0.0002672915074176452, "loss_nats": 1.2969269752502441, "step": 2120, "unclipped_grad_norm": 0.7505779266357422 }, { "bits_per_byte": 1.8349618420920515, "learning_rate": 0.00026692646159965624, "loss_nats": 1.271898627281189, "step": 2130, "unclipped_grad_norm": 0.7835680246353149 }, { "bits_per_byte": 1.9616896661374241, "learning_rate": 0.0002665596741396633, "loss_nats": 1.3597396612167358, "step": 2140, "unclipped_grad_norm": 0.7786996960639954 }, { "bits_per_byte": 1.8168395142226847, "learning_rate": 0.000266191151305052, "loss_nats": 1.2593371868133545, "step": 2150, "unclipped_grad_norm": 0.7194197773933411 }, { "bits_per_byte": 1.8739360344826286, "learning_rate": 0.00026582089939286075, "loss_nats": 1.2989134788513184, "step": 2160, "unclipped_grad_norm": 0.8012002110481262 }, { "bits_per_byte": 1.9634492206384013, "learning_rate": 0.0002654489247296732, "loss_nats": 1.3609592914581299, "step": 2170, "unclipped_grad_norm": 0.8270084261894226 }, { "bits_per_byte": 1.8175344961148212, "learning_rate": 0.00026507523367151007, "loss_nats": 1.2598189115524292, "step": 2180, "unclipped_grad_norm": 0.7824540734291077 }, { "bits_per_byte": 1.8458584908680682, "learning_rate": 0.00026469983260372037, "loss_nats": 1.279451608657837, "step": 2190, "unclipped_grad_norm": 0.7973959445953369 }, { "bits_per_byte": 1.8557901449900935, "learning_rate": 0.0002643227279408727, "loss_nats": 1.2863357067108154, "step": 2200, "unclipped_grad_norm": 0.7451788783073425 }, { "bits_per_byte": 1.8380872828061812, "learning_rate": 0.0002639439261266453, "loss_nats": 1.2740650177001953, "step": 2210, "unclipped_grad_norm": 0.7804975509643555 }, { "bits_per_byte": 1.8298283319464475, "learning_rate": 0.000263563433633716, "loss_nats": 1.2683403491973877, "step": 2220, "unclipped_grad_norm": 0.7694668173789978 }, { "bits_per_byte": 1.9031531351418058, "learning_rate": 0.00026318125696365156, "loss_nats": 1.3191652297973633, "step": 2230, "unclipped_grad_norm": 0.8418469429016113 }, { "bits_per_byte": 1.8400618156207371, "learning_rate": 0.000262797402646797, "loss_nats": 1.2754336595535278, "step": 2240, "unclipped_grad_norm": 0.7987614870071411 }, { "bits_per_byte": 1.852736936989326, "learning_rate": 0.00026241187724216334, "loss_nats": 1.2842193841934204, "step": 2250, "unclipped_grad_norm": 0.7795421481132507 }, { "bits_per_byte": 1.8339105121473271, "learning_rate": 0.0002620246873373162, "loss_nats": 1.271169900894165, "step": 2260, "unclipped_grad_norm": 0.7681040167808533 }, { "bits_per_byte": 1.825916414569922, "learning_rate": 0.00026163583954826264, "loss_nats": 1.2656288146972656, "step": 2270, "unclipped_grad_norm": 0.7933830618858337 }, { "bits_per_byte": 1.8451832869807807, "learning_rate": 0.0002612453405193386, "loss_nats": 1.2789835929870605, "step": 2280, "unclipped_grad_norm": 0.7935850024223328 }, { "bits_per_byte": 1.8711039961708706, "learning_rate": 0.000260853196923095, "loss_nats": 1.2969504594802856, "step": 2290, "unclipped_grad_norm": 0.7818295955657959 }, { "bits_per_byte": 1.7944778140095496, "learning_rate": 0.0002604594154601839, "loss_nats": 1.2438372373580933, "step": 2300, "unclipped_grad_norm": 0.7245032787322998 }, { "bits_per_byte": 1.8221434591752936, "learning_rate": 0.0002600640028592441, "loss_nats": 1.2630136013031006, "step": 2310, "unclipped_grad_norm": 0.7396021485328674 }, { "bits_per_byte": 1.7928696042413317, "learning_rate": 0.0002596669658767859, "loss_nats": 1.242722511291504, "step": 2320, "unclipped_grad_norm": 0.7618010640144348 }, { "bits_per_byte": 1.8702191454321815, "learning_rate": 0.000259268311297076, "loss_nats": 1.2963371276855469, "step": 2330, "unclipped_grad_norm": 0.7263991236686707 }, { "bits_per_byte": 1.7994940319872652, "learning_rate": 0.00025886804593202114, "loss_nats": 1.247314214706421, "step": 2340, "unclipped_grad_norm": 0.7788637280464172 }, { "bits_per_byte": 1.849551646312696, "learning_rate": 0.00025846617662105193, "loss_nats": 1.2820115089416504, "step": 2350, "unclipped_grad_norm": 0.7732409238815308 }, { "bits_per_byte": 1.8613248906601605, "learning_rate": 0.00025806271023100637, "loss_nats": 1.2901721000671387, "step": 2360, "unclipped_grad_norm": 0.8177725672721863 }, { "bits_per_byte": 1.8822589628884834, "learning_rate": 0.00025765765365601165, "loss_nats": 1.3046824932098389, "step": 2370, "unclipped_grad_norm": 0.7931574583053589 }, { "bits_per_byte": 1.8382422391746087, "learning_rate": 0.00025725101381736707, "loss_nats": 1.2741724252700806, "step": 2380, "unclipped_grad_norm": 0.8604201674461365 }, { "bits_per_byte": 1.8797817247854554, "learning_rate": 0.00025684279766342567, "loss_nats": 1.3029654026031494, "step": 2390, "unclipped_grad_norm": 0.7636454105377197 }, { "bits_per_byte": 1.817860919186159, "learning_rate": 0.00025643301216947504, "loss_nats": 1.2600451707839966, "step": 2400, "unclipped_grad_norm": 0.779283881187439 }, { "bits_per_byte": 1.8354538844561699, "learning_rate": 0.00025602166433761866, "loss_nats": 1.2722396850585938, "step": 2410, "unclipped_grad_norm": 0.7781111001968384 }, { "bits_per_byte": 1.8069649583407457, "learning_rate": 0.0002556087611966561, "loss_nats": 1.2524926662445068, "step": 2420, "unclipped_grad_norm": 0.7548633813858032 }, { "bits_per_byte": 1.8169932667125561, "learning_rate": 0.00025519430980196277, "loss_nats": 1.259443759918213, "step": 2430, "unclipped_grad_norm": 0.7562755942344666 }, { "bits_per_byte": 1.8760644917697067, "learning_rate": 0.0002547783172353695, "loss_nats": 1.3003888130187988, "step": 2440, "unclipped_grad_norm": 0.7478408813476562 }, { "bits_per_byte": 1.838183937055966, "learning_rate": 0.0002543607906050415, "loss_nats": 1.2741320133209229, "step": 2450, "unclipped_grad_norm": 0.760445237159729 }, { "bits_per_byte": 1.8195946762895068, "learning_rate": 0.0002539417370453568, "loss_nats": 1.261246919631958, "step": 2460, "unclipped_grad_norm": 0.7331485748291016 }, { "bits_per_byte": 1.8163858239897086, "learning_rate": 0.00025352116371678464, "loss_nats": 1.2590227127075195, "step": 2470, "unclipped_grad_norm": 0.7682369351387024 }, { "bits_per_byte": 1.8266683227194942, "learning_rate": 0.0002530990778057627, "loss_nats": 1.2661499977111816, "step": 2480, "unclipped_grad_norm": 0.7132375240325928 }, { "bits_per_byte": 1.8042156436840528, "learning_rate": 0.00025267548652457475, "loss_nats": 1.250586986541748, "step": 2490, "unclipped_grad_norm": 0.7770085334777832 }, { "bits_per_byte": 1.8313916542427904, "learning_rate": 0.00025225039711122696, "loss_nats": 1.2694239616394043, "step": 2500, "unclipped_grad_norm": 0.7582318782806396 }, { "bits_per_byte": 1.7788244690759694, "learning_rate": 0.00025182381682932464, "loss_nats": 1.2329871654510498, "step": 2510, "unclipped_grad_norm": 0.8006963133811951 }, { "bits_per_byte": 1.8448090527325032, "learning_rate": 0.0002513957529679477, "loss_nats": 1.278724193572998, "step": 2520, "unclipped_grad_norm": 0.7802237272262573 }, { "bits_per_byte": 1.855150369528884, "learning_rate": 0.00025096621284152656, "loss_nats": 1.2858922481536865, "step": 2530, "unclipped_grad_norm": 0.7603861689567566 }, { "bits_per_byte": 1.7974587892969547, "learning_rate": 0.0002505352037897168, "loss_nats": 1.245903491973877, "step": 2540, "unclipped_grad_norm": 0.7829954624176025 }, { "bits_per_byte": 1.8048358131230642, "learning_rate": 0.0002501027331772739, "loss_nats": 1.2510168552398682, "step": 2550, "unclipped_grad_norm": 0.8207260966300964 }, { "bits_per_byte": 1.8410618947355044, "learning_rate": 0.00024966880839392746, "loss_nats": 1.2761268615722656, "step": 2560, "unclipped_grad_norm": 0.8064310550689697 }, { "bits_per_byte": 1.8072459779922554, "learning_rate": 0.0002492334368542548, "loss_nats": 1.2526874542236328, "step": 2570, "unclipped_grad_norm": 0.7300699353218079 }, { "bits_per_byte": 1.8352736466380657, "learning_rate": 0.00024879662599755416, "loss_nats": 1.2721147537231445, "step": 2580, "unclipped_grad_norm": 0.7476873993873596 }, { "bits_per_byte": 1.793617556789934, "learning_rate": 0.00024835838328771803, "loss_nats": 1.2432409524917603, "step": 2590, "unclipped_grad_norm": 0.7210070490837097 }, { "bits_per_byte": 1.797604974550188, "learning_rate": 0.0002479187162131051, "loss_nats": 1.2460048198699951, "step": 2600, "unclipped_grad_norm": 0.7853913903236389 }, { "bits_per_byte": 1.7385629865436842, "learning_rate": 0.00024747763228641264, "loss_nats": 1.2050800323486328, "step": 2610, "unclipped_grad_norm": 0.7092626094818115 }, { "bits_per_byte": 1.8079149905041119, "learning_rate": 0.00024703513904454795, "loss_nats": 1.2531511783599854, "step": 2620, "unclipped_grad_norm": 0.7522015571594238 }, { "bits_per_byte": 1.8198622812942493, "learning_rate": 0.00024659124404849956, "loss_nats": 1.261432409286499, "step": 2630, "unclipped_grad_norm": 0.7675716876983643 }, { "bits_per_byte": 1.680096624469618, "learning_rate": 0.00024614595488320833, "loss_nats": 1.164554238319397, "step": 2640, "unclipped_grad_norm": 0.6825466156005859 }, { "bits_per_byte": 1.7638260340768748, "learning_rate": 0.00024569927915743745, "loss_nats": 1.2225910425186157, "step": 2650, "unclipped_grad_norm": 0.7630164623260498 }, { "bits_per_byte": 1.8201589513669878, "learning_rate": 0.0002452512245036427, "loss_nats": 1.2616380453109741, "step": 2660, "unclipped_grad_norm": 0.7772946953773499 }, { "bits_per_byte": 1.811244230659514, "learning_rate": 0.00024480179857784177, "loss_nats": 1.2554588317871094, "step": 2670, "unclipped_grad_norm": 0.7582975625991821 }, { "bits_per_byte": 1.74257860945868, "learning_rate": 0.00024435100905948387, "loss_nats": 1.207863450050354, "step": 2680, "unclipped_grad_norm": 0.7818799614906311 }, { "bits_per_byte": 1.9030688636428832, "learning_rate": 0.00024389886365131793, "loss_nats": 1.3191068172454834, "step": 2690, "unclipped_grad_norm": 0.7075229287147522 }, { "bits_per_byte": 1.8200497423836899, "learning_rate": 0.00024344537007926163, "loss_nats": 1.2615623474121094, "step": 2700, "unclipped_grad_norm": 0.8272333741188049 }, { "bits_per_byte": 1.7354167359288, "learning_rate": 0.00024299053609226878, "loss_nats": 1.2028992176055908, "step": 2710, "unclipped_grad_norm": 0.7310962677001953 }, { "bits_per_byte": 1.728331394678554, "learning_rate": 0.00024253436946219733, "loss_nats": 1.1979880332946777, "step": 2720, "unclipped_grad_norm": 0.7193791270256042 }, { "bits_per_byte": 1.76859700479446, "learning_rate": 0.0002420768779836764, "loss_nats": 1.225898027420044, "step": 2730, "unclipped_grad_norm": 0.773657500743866 }, { "bits_per_byte": 1.7790022991369616, "learning_rate": 0.00024161806947397305, "loss_nats": 1.2331104278564453, "step": 2740, "unclipped_grad_norm": 0.7759174704551697 }, { "bits_per_byte": 1.7909771071512373, "learning_rate": 0.00024115795177285886, "loss_nats": 1.241410732269287, "step": 2750, "unclipped_grad_norm": 0.770866870880127 }, { "bits_per_byte": 1.81886375002334, "learning_rate": 0.0002406965327424758, "loss_nats": 1.2607402801513672, "step": 2760, "unclipped_grad_norm": 0.7793042659759521 }, { "bits_per_byte": 1.704546881994831, "learning_rate": 0.00024023382026720197, "loss_nats": 1.181501865386963, "step": 2770, "unclipped_grad_norm": 0.7179723978042603 }, { "bits_per_byte": 1.7222445846992174, "learning_rate": 0.00023976982225351698, "loss_nats": 1.1937689781188965, "step": 2780, "unclipped_grad_norm": 0.711125910282135 }, { "bits_per_byte": 1.8073385046584196, "learning_rate": 0.0002393045466298666, "loss_nats": 1.2527515888214111, "step": 2790, "unclipped_grad_norm": 0.7564381957054138 }, { "bits_per_byte": 1.8564130661515186, "learning_rate": 0.00023883800134652763, "loss_nats": 1.2867674827575684, "step": 2800, "unclipped_grad_norm": 0.7982874512672424 }, { "bits_per_byte": 1.7924368958917607, "learning_rate": 0.0002383701943754715, "loss_nats": 1.2424225807189941, "step": 2810, "unclipped_grad_norm": 0.827074408531189 }, { "bits_per_byte": 1.781587886310033, "learning_rate": 0.00023790113371022884, "loss_nats": 1.2349026203155518, "step": 2820, "unclipped_grad_norm": 0.6945767998695374 }, { "bits_per_byte": 1.862601517877515, "learning_rate": 0.0002374308273657521, "loss_nats": 1.2910569906234741, "step": 2830, "unclipped_grad_norm": 0.7473636269569397 }, { "bits_per_byte": 1.7520161574247766, "learning_rate": 0.00023695928337827911, "loss_nats": 1.2144050598144531, "step": 2840, "unclipped_grad_norm": 0.7269325852394104 }, { "bits_per_byte": 1.7726036846116109, "learning_rate": 0.00023648650980519564, "loss_nats": 1.2286752462387085, "step": 2850, "unclipped_grad_norm": 0.7537488341331482 }, { "bits_per_byte": 1.7375121725469125, "learning_rate": 0.00023601251472489757, "loss_nats": 1.2043516635894775, "step": 2860, "unclipped_grad_norm": 0.7839256525039673 }, { "bits_per_byte": 1.836773851301542, "learning_rate": 0.00023553730623665305, "loss_nats": 1.273154616355896, "step": 2870, "unclipped_grad_norm": 0.7615643739700317 }, { "bits_per_byte": 1.8235200083127996, "learning_rate": 0.00023506089246046403, "loss_nats": 1.263967752456665, "step": 2880, "unclipped_grad_norm": 0.7500733137130737 }, { "bits_per_byte": 1.7346435019305209, "learning_rate": 0.00023458328153692736, "loss_nats": 1.2023632526397705, "step": 2890, "unclipped_grad_norm": 0.7335168123245239 }, { "bits_per_byte": 1.710140445731493, "learning_rate": 0.0002341044816270961, "loss_nats": 1.1853790283203125, "step": 2900, "unclipped_grad_norm": 0.7237004041671753 }, { "bits_per_byte": 1.7640040361205178, "learning_rate": 0.00023362450091233953, "loss_nats": 1.2227144241333008, "step": 2910, "unclipped_grad_norm": 0.7653093338012695 }, { "bits_per_byte": 1.7372325287566095, "learning_rate": 0.00023314334759420388, "loss_nats": 1.204157829284668, "step": 2920, "unclipped_grad_norm": 0.7592934966087341 }, { "bits_per_byte": 1.7605243111456106, "learning_rate": 0.0002326610298942717, "loss_nats": 1.2203024625778198, "step": 2930, "unclipped_grad_norm": 0.7871323227882385 }, { "bits_per_byte": 1.7773966691085064, "learning_rate": 0.00023217755605402175, "loss_nats": 1.2319974899291992, "step": 2940, "unclipped_grad_norm": 0.7994735240936279 }, { "bits_per_byte": 1.761262116717812, "learning_rate": 0.00023169293433468802, "loss_nats": 1.2208138704299927, "step": 2950, "unclipped_grad_norm": 0.7672474384307861 }, { "bits_per_byte": 1.8009139207527887, "learning_rate": 0.00023120717301711857, "loss_nats": 1.2482984066009521, "step": 2960, "unclipped_grad_norm": 0.7721540927886963 }, { "bits_per_byte": 1.7577087831683356, "learning_rate": 0.00023072028040163408, "loss_nats": 1.218350887298584, "step": 2970, "unclipped_grad_norm": 0.7659915685653687 }, { "bits_per_byte": 1.7511909846659368, "learning_rate": 0.00023023226480788593, "loss_nats": 1.2138330936431885, "step": 2980, "unclipped_grad_norm": 0.7457980513572693 }, { "bits_per_byte": 1.7677185174138528, "learning_rate": 0.00022974313457471416, "loss_nats": 1.2252891063690186, "step": 2990, "unclipped_grad_norm": 0.7551211714744568 }, { "bits_per_byte": 1.7804380103063644, "learning_rate": 0.0002292528980600049, "loss_nats": 1.2341055870056152, "step": 3000, "unclipped_grad_norm": 0.7369211316108704 }, { "bits_per_byte": 1.77081558099059, "learning_rate": 0.00022876156364054758, "loss_nats": 1.227435827255249, "step": 3010, "unclipped_grad_norm": 0.7526711225509644 }, { "bits_per_byte": 1.726040929734363, "learning_rate": 0.0002282691397118918, "loss_nats": 1.1964004039764404, "step": 3020, "unclipped_grad_norm": 0.8113403916358948 }, { "bits_per_byte": 1.8045116258261877, "learning_rate": 0.00022777563468820376, "loss_nats": 1.250792145729065, "step": 3030, "unclipped_grad_norm": 0.7493385672569275 }, { "bits_per_byte": 1.7229166928987893, "learning_rate": 0.00022728105700212282, "loss_nats": 1.194234848022461, "step": 3040, "unclipped_grad_norm": 0.7606903314590454 }, { "bits_per_byte": 1.7403560776615798, "learning_rate": 0.00022678541510461698, "loss_nats": 1.2063229084014893, "step": 3050, "unclipped_grad_norm": 0.7690193057060242 }, { "bits_per_byte": 1.759874388708, "learning_rate": 0.00022628871746483876, "loss_nats": 1.2198519706726074, "step": 3060, "unclipped_grad_norm": 0.7480750679969788 }, { "bits_per_byte": 1.6873952242069339, "learning_rate": 0.0002257909725699804, "loss_nats": 1.169613242149353, "step": 3070, "unclipped_grad_norm": 0.7001752853393555 }, { "bits_per_byte": 1.737272084766308, "learning_rate": 0.00022529218892512886, "loss_nats": 1.2041852474212646, "step": 3080, "unclipped_grad_norm": 0.8322204351425171 }, { "bits_per_byte": 1.8015143121869506, "learning_rate": 0.00022479237505312034, "loss_nats": 1.248714566230774, "step": 3090, "unclipped_grad_norm": 0.8183287382125854 }, { "bits_per_byte": 1.7001176408045102, "learning_rate": 0.00022429153949439496, "loss_nats": 1.178431749343872, "step": 3100, "unclipped_grad_norm": 0.7750749588012695 }, { "bits_per_byte": 1.7770231227908324, "learning_rate": 0.00022378969080685053, "loss_nats": 1.231738567352295, "step": 3110, "unclipped_grad_norm": 0.7082787752151489 }, { "bits_per_byte": 1.749435729731231, "learning_rate": 0.0002232868375656965, "loss_nats": 1.2126164436340332, "step": 3120, "unclipped_grad_norm": 0.736611545085907 }, { "bits_per_byte": 1.7966217497352055, "learning_rate": 0.00022278298836330733, "loss_nats": 1.2453233003616333, "step": 3130, "unclipped_grad_norm": 0.7366204857826233 }, { "bits_per_byte": 1.7636881039908827, "learning_rate": 0.00022227815180907572, "loss_nats": 1.222495436668396, "step": 3140, "unclipped_grad_norm": 0.7583194375038147 }, { "bits_per_byte": 1.7359698321339747, "learning_rate": 0.00022177233652926546, "loss_nats": 1.2032825946807861, "step": 3150, "unclipped_grad_norm": 0.7600930333137512 }, { "bits_per_byte": 1.7415977924008093, "learning_rate": 0.00022126555116686416, "loss_nats": 1.207183599472046, "step": 3160, "unclipped_grad_norm": 0.7639689445495605 }, { "bits_per_byte": 1.654884311818432, "learning_rate": 0.00022075780438143532, "loss_nats": 1.1470783948898315, "step": 3170, "unclipped_grad_norm": 0.7011478543281555 }, { "bits_per_byte": 1.7584833930278216, "learning_rate": 0.00022024910484897065, "loss_nats": 1.2188878059387207, "step": 3180, "unclipped_grad_norm": 0.7645173072814941 }, { "bits_per_byte": 1.7305350083840592, "learning_rate": 0.00021973946126174158, "loss_nats": 1.199515461921692, "step": 3190, "unclipped_grad_norm": 0.7665988206863403 }, { "bits_per_byte": 1.7003714871971838, "learning_rate": 0.00021922888232815086, "loss_nats": 1.178607702255249, "step": 3200, "unclipped_grad_norm": 0.7379894852638245 }, { "bits_per_byte": 1.6959066455981344, "learning_rate": 0.0002187173767725837, "loss_nats": 1.1755129098892212, "step": 3210, "unclipped_grad_norm": 0.8013631701469421 }, { "bits_per_byte": 1.7098007799960389, "learning_rate": 0.0002182049533352588, "loss_nats": 1.1851435899734497, "step": 3220, "unclipped_grad_norm": 0.7296364903450012 }, { "bits_per_byte": 1.7987765203678656, "learning_rate": 0.00021769162077207884, "loss_nats": 1.246816873550415, "step": 3230, "unclipped_grad_norm": 0.7610434889793396 }, { "bits_per_byte": 1.6862874839527267, "learning_rate": 0.00021717738785448102, "loss_nats": 1.1688454151153564, "step": 3240, "unclipped_grad_norm": 0.723595917224884 }, { "bits_per_byte": 1.8345631863073515, "learning_rate": 0.00021666226336928708, "loss_nats": 1.2716223001480103, "step": 3250, "unclipped_grad_norm": 0.8353017568588257 }, { "bits_per_byte": 1.6872093109613513, "learning_rate": 0.0002161462561185532, "loss_nats": 1.1694843769073486, "step": 3260, "unclipped_grad_norm": 0.7602720260620117 }, { "bits_per_byte": 1.7198954736710823, "learning_rate": 0.00021562937491941957, "loss_nats": 1.1921406984329224, "step": 3270, "unclipped_grad_norm": 0.7516167163848877 }, { "bits_per_byte": 1.7692164863028679, "learning_rate": 0.00021511162860395975, "loss_nats": 1.2263274192810059, "step": 3280, "unclipped_grad_norm": 0.7487898468971252 }, { "bits_per_byte": 1.7624494849393688, "learning_rate": 0.0002145930260190298, "loss_nats": 1.2216368913650513, "step": 3290, "unclipped_grad_norm": 0.7708044052124023 }, { "bits_per_byte": 1.7002387165907176, "learning_rate": 0.00021407357602611698, "loss_nats": 1.1785156726837158, "step": 3300, "unclipped_grad_norm": 0.7703046202659607 }, { "bits_per_byte": 1.7332599015043295, "learning_rate": 0.0002135532875011885, "loss_nats": 1.2014042139053345, "step": 3310, "unclipped_grad_norm": 0.8132672905921936 }, { "bits_per_byte": 1.7262779218272521, "learning_rate": 0.0002130321693345397, "loss_nats": 1.1965646743774414, "step": 3320, "unclipped_grad_norm": 0.7545798420906067 }, { "bits_per_byte": 1.7272033604715449, "learning_rate": 0.00021251023043064224, "loss_nats": 1.1972061395645142, "step": 3330, "unclipped_grad_norm": 0.7775130867958069 }, { "bits_per_byte": 1.6975052243379043, "learning_rate": 0.00021198747970799188, "loss_nats": 1.1766209602355957, "step": 3340, "unclipped_grad_norm": 0.7105540633201599 }, { "bits_per_byte": 1.7083408192728646, "learning_rate": 0.00021146392609895618, "loss_nats": 1.1841316223144531, "step": 3350, "unclipped_grad_norm": 0.7163253426551819 }, { "bits_per_byte": 1.732425613664994, "learning_rate": 0.00021093957854962178, "loss_nats": 1.2008259296417236, "step": 3360, "unclipped_grad_norm": 0.7626740336418152 }, { "bits_per_byte": 1.6756070173688444, "learning_rate": 0.00021041444601964157, "loss_nats": 1.1614422798156738, "step": 3370, "unclipped_grad_norm": 0.8061453700065613 }, { "bits_per_byte": 1.7092863798873084, "learning_rate": 0.00020988853748208164, "loss_nats": 1.1847870349884033, "step": 3380, "unclipped_grad_norm": 0.7406735420227051 }, { "bits_per_byte": 1.7539413312185348, "learning_rate": 0.00020936186192326782, "loss_nats": 1.2157394886016846, "step": 3390, "unclipped_grad_norm": 0.7695116400718689 }, { "bits_per_byte": 1.8092177590843976, "learning_rate": 0.00020883442834263225, "loss_nats": 1.2540541887283325, "step": 3400, "unclipped_grad_norm": 0.7858016490936279 }, { "bits_per_byte": 1.8255297975707825, "learning_rate": 0.0002083062457525596, "loss_nats": 1.2653608322143555, "step": 3410, "unclipped_grad_norm": 0.7603964805603027 }, { "bits_per_byte": 1.6857877023693193, "learning_rate": 0.00020777732317823306, "loss_nats": 1.1684989929199219, "step": 3420, "unclipped_grad_norm": 0.7168062329292297 }, { "bits_per_byte": 1.6176447364444573, "learning_rate": 0.00020724766965748005, "loss_nats": 1.1212658882141113, "step": 3430, "unclipped_grad_norm": 0.7431134581565857 }, { "bits_per_byte": 1.711189711884407, "learning_rate": 0.00020671729424061788, "loss_nats": 1.1861063241958618, "step": 3440, "unclipped_grad_norm": 0.7753933668136597 }, { "bits_per_byte": 1.77535626693867, "learning_rate": 0.00020618620599029906, "loss_nats": 1.2305831909179688, "step": 3450, "unclipped_grad_norm": 0.8042635321617126 }, { "bits_per_byte": 1.7083845028661837, "learning_rate": 0.00020565441398135649, "loss_nats": 1.184161901473999, "step": 3460, "unclipped_grad_norm": 0.7639697194099426 }, { "bits_per_byte": 1.6937178224006033, "learning_rate": 0.00020512192730064827, "loss_nats": 1.1739957332611084, "step": 3470, "unclipped_grad_norm": 0.748127281665802 }, { "bits_per_byte": 1.6501064617948125, "learning_rate": 0.00020458875504690273, "loss_nats": 1.1437666416168213, "step": 3480, "unclipped_grad_norm": 0.7506517767906189 }, { "bits_per_byte": 1.8358463488654388, "learning_rate": 0.00020405490633056245, "loss_nats": 1.2725117206573486, "step": 3490, "unclipped_grad_norm": 0.836082398891449 }, { "bits_per_byte": 1.714104301868579, "learning_rate": 0.00020352039027362917, "loss_nats": 1.188126564025879, "step": 3500, "unclipped_grad_norm": 0.7544166445732117 }, { "bits_per_byte": 1.744907598516664, "learning_rate": 0.0002029852160095075, "loss_nats": 1.2094777822494507, "step": 3510, "unclipped_grad_norm": 0.7650617361068726 }, { "bits_per_byte": 1.738600994709525, "learning_rate": 0.000202449392682849, "loss_nats": 1.2051063776016235, "step": 3520, "unclipped_grad_norm": 0.7794367671012878 }, { "bits_per_byte": 1.7051804660806096, "learning_rate": 0.0002019129294493959, "loss_nats": 1.181941032409668, "step": 3530, "unclipped_grad_norm": 0.7445787787437439 }, { "bits_per_byte": 1.7158172490711725, "learning_rate": 0.00020137583547582478, "loss_nats": 1.1893138885498047, "step": 3540, "unclipped_grad_norm": 0.7629107236862183 }, { "bits_per_byte": 1.7173026632266746, "learning_rate": 0.00020083811993958973, "loss_nats": 1.1903434991836548, "step": 3550, "unclipped_grad_norm": 0.7763495445251465 }, { "bits_per_byte": 1.6560372835098163, "learning_rate": 0.00020029979202876563, "loss_nats": 1.14787757396698, "step": 3560, "unclipped_grad_norm": 0.7450886964797974 }, { "bits_per_byte": 1.6838996768281473, "learning_rate": 0.00019976086094189118, "loss_nats": 1.1671903133392334, "step": 3570, "unclipped_grad_norm": 0.7487124800682068 }, { "bits_per_byte": 1.6946102403759304, "learning_rate": 0.00019922133588781175, "loss_nats": 1.1746143102645874, "step": 3580, "unclipped_grad_norm": 0.757170557975769 }, { "bits_per_byte": 1.7307594457434352, "learning_rate": 0.0001986812260855219, "loss_nats": 1.1996710300445557, "step": 3590, "unclipped_grad_norm": 0.7700981497764587 }, { "bits_per_byte": 1.7491381997452384, "learning_rate": 0.00019814054076400787, "loss_nats": 1.2124102115631104, "step": 3600, "unclipped_grad_norm": 0.7877120971679688 }, { "bits_per_byte": 1.688025540622346, "learning_rate": 0.00019759928916209004, "loss_nats": 1.1700501441955566, "step": 3610, "unclipped_grad_norm": 0.780867874622345 }, { "bits_per_byte": 1.6436277033541595, "learning_rate": 0.00019705748052826493, "loss_nats": 1.1392759084701538, "step": 3620, "unclipped_grad_norm": 0.7423334717750549 }, { "bits_per_byte": 1.6950637586262558, "learning_rate": 0.00019651512412054723, "loss_nats": 1.1749286651611328, "step": 3630, "unclipped_grad_norm": 0.7513464689254761 }, { "bits_per_byte": 1.6758522646289749, "learning_rate": 0.00019597222920631144, "loss_nats": 1.1616122722625732, "step": 3640, "unclipped_grad_norm": 0.7453336715698242 }, { "bits_per_byte": 1.736063734661346, "learning_rate": 0.0001954288050621338, "loss_nats": 1.2033476829528809, "step": 3650, "unclipped_grad_norm": 0.7682862877845764 }, { "bits_per_byte": 1.6360068081291268, "learning_rate": 0.00019488486097363363, "loss_nats": 1.1339935064315796, "step": 3660, "unclipped_grad_norm": 0.7531437277793884 }, { "bits_per_byte": 1.7183529612754938, "learning_rate": 0.00019434040623531452, "loss_nats": 1.1910715103149414, "step": 3670, "unclipped_grad_norm": 0.7578480243682861 }, { "bits_per_byte": 1.6583936178092877, "learning_rate": 0.0001937954501504058, "loss_nats": 1.1495108604431152, "step": 3680, "unclipped_grad_norm": 0.7224898934364319 }, { "bits_per_byte": 1.6372521345040243, "learning_rate": 0.00019325000203070333, "loss_nats": 1.1348567008972168, "step": 3690, "unclipped_grad_norm": 0.7246464490890503 }, { "bits_per_byte": 1.6659130432703113, "learning_rate": 0.0001927040711964106, "loss_nats": 1.1547229290008545, "step": 3700, "unclipped_grad_norm": 0.7878638505935669 }, { "bits_per_byte": 1.694639821391879, "learning_rate": 0.00019215766697597923, "loss_nats": 1.1746348142623901, "step": 3710, "unclipped_grad_norm": 0.7455682754516602 }, { "bits_per_byte": 1.6891824679147003, "learning_rate": 0.00019161079870594978, "loss_nats": 1.1708520650863647, "step": 3720, "unclipped_grad_norm": 0.7515487670898438 }, { "bits_per_byte": 1.6091924770851538, "learning_rate": 0.0001910634757307922, "loss_nats": 1.1154072284698486, "step": 3730, "unclipped_grad_norm": 0.7353127598762512 }, { "bits_per_byte": 1.6633680440028435, "learning_rate": 0.0001905157074027459, "loss_nats": 1.152958869934082, "step": 3740, "unclipped_grad_norm": 0.7253232002258301 }, { "bits_per_byte": 1.6922951819126664, "learning_rate": 0.00018996750308166037, "loss_nats": 1.1730096340179443, "step": 3750, "unclipped_grad_norm": 0.7686370611190796 }, { "bits_per_byte": 1.7713081393026613, "learning_rate": 0.00018941887213483482, "loss_nats": 1.2277772426605225, "step": 3760, "unclipped_grad_norm": 0.7601476907730103 }, { "bits_per_byte": 1.7305630415561497, "learning_rate": 0.00018886982393685845, "loss_nats": 1.1995348930358887, "step": 3770, "unclipped_grad_norm": 0.7877833247184753 }, { "bits_per_byte": 1.5644979697442287, "learning_rate": 0.00018832036786945008, "loss_nats": 1.0844273567199707, "step": 3780, "unclipped_grad_norm": 0.7386898994445801 }, { "bits_per_byte": 1.6644766441703052, "learning_rate": 0.0001877705133212979, "loss_nats": 1.1537272930145264, "step": 3790, "unclipped_grad_norm": 0.7420179843902588 }, { "bits_per_byte": 1.6653939996300076, "learning_rate": 0.00018722026968789906, "loss_nats": 1.1543631553649902, "step": 3800, "unclipped_grad_norm": 0.7838358283042908 }, { "bits_per_byte": 1.6952983429620325, "learning_rate": 0.00018666964637139907, "loss_nats": 1.17509126663208, "step": 3810, "unclipped_grad_norm": 0.8048396110534668 }, { "bits_per_byte": 1.6744323758634516, "learning_rate": 0.00018611865278043115, "loss_nats": 1.160628080368042, "step": 3820, "unclipped_grad_norm": 0.7709245681762695 }, { "bits_per_byte": 1.720191799778519, "learning_rate": 0.00018556729832995555, "loss_nats": 1.1923460960388184, "step": 3830, "unclipped_grad_norm": 0.7672807574272156 }, { "bits_per_byte": 1.6467005173771254, "learning_rate": 0.00018501559244109864, "loss_nats": 1.1414058208465576, "step": 3840, "unclipped_grad_norm": 0.7319741249084473 }, { "bits_per_byte": 1.5964396195583748, "learning_rate": 0.00018446354454099178, "loss_nats": 1.106567621231079, "step": 3850, "unclipped_grad_norm": 0.7756319642066956 }, { "bits_per_byte": 1.6360811046342996, "learning_rate": 0.00018391116406261048, "loss_nats": 1.1340450048446655, "step": 3860, "unclipped_grad_norm": 0.766754150390625 }, { "bits_per_byte": 1.7433616464680584, "learning_rate": 0.00018335846044461305, "loss_nats": 1.2084062099456787, "step": 3870, "unclipped_grad_norm": 0.7916337847709656 }, { "bits_per_byte": 1.6358671582166262, "learning_rate": 0.0001828054431311793, "loss_nats": 1.1338967084884644, "step": 3880, "unclipped_grad_norm": 0.7497678399085999 }, { "bits_per_byte": 1.600699113872294, "learning_rate": 0.00018225212157184934, "loss_nats": 1.1095200777053833, "step": 3890, "unclipped_grad_norm": 0.7555440664291382 }, { "bits_per_byte": 1.6778517349279058, "learning_rate": 0.00018169850522136192, "loss_nats": 1.1629981994628906, "step": 3900, "unclipped_grad_norm": 0.826478123664856 }, { "bits_per_byte": 1.6805711246033486, "learning_rate": 0.000181144603539493, "loss_nats": 1.1648831367492676, "step": 3910, "unclipped_grad_norm": 0.7849931120872498 }, { "bits_per_byte": 1.6375147520118918, "learning_rate": 0.00018059042599089412, "loss_nats": 1.1350387334823608, "step": 3920, "unclipped_grad_norm": 0.7588120102882385 }, { "bits_per_byte": 1.673005091843941, "learning_rate": 0.00018003598204493048, "loss_nats": 1.15963876247406, "step": 3930, "unclipped_grad_norm": 0.742434024810791 }, { "bits_per_byte": 1.6996766772876981, "learning_rate": 0.00017948128117551936, "loss_nats": 1.1781260967254639, "step": 3940, "unclipped_grad_norm": 0.7588658928871155 }, { "bits_per_byte": 1.7339400928884918, "learning_rate": 0.00017892633286096816, "loss_nats": 1.2018756866455078, "step": 3950, "unclipped_grad_norm": 0.7937244176864624 }, { "bits_per_byte": 1.713467450112434, "learning_rate": 0.00017837114658381247, "loss_nats": 1.1876851320266724, "step": 3960, "unclipped_grad_norm": 0.7884036302566528 }, { "bits_per_byte": 1.6516962694193882, "learning_rate": 0.00017781573183065386, "loss_nats": 1.1448686122894287, "step": 3970, "unclipped_grad_norm": 0.8004661798477173 }, { "bits_per_byte": 1.6740500584305835, "learning_rate": 0.0001772600980919981, "loss_nats": 1.1603630781173706, "step": 3980, "unclipped_grad_norm": 0.7692103385925293 }, { "bits_per_byte": 1.691941929547794, "learning_rate": 0.0001767042548620927, "loss_nats": 1.172764778137207, "step": 3990, "unclipped_grad_norm": 0.7891059517860413 }, { "bits_per_byte": 1.5813677479673616, "learning_rate": 0.00017614821163876486, "loss_nats": 1.0961205959320068, "step": 4000, "unclipped_grad_norm": 0.7436221837997437 }, { "bits_per_byte": 1.6793110077204767, "learning_rate": 0.00017559197792325906, "loss_nats": 1.164009690284729, "step": 4010, "unclipped_grad_norm": 0.8044730424880981 }, { "bits_per_byte": 1.6584589712166156, "learning_rate": 0.0001750355632200748, "loss_nats": 1.1495561599731445, "step": 4020, "unclipped_grad_norm": 0.8183668851852417 }, { "bits_per_byte": 1.6850787898824628, "learning_rate": 0.00017447897703680418, "loss_nats": 1.1680076122283936, "step": 4030, "unclipped_grad_norm": 0.793128252029419 }, { "bits_per_byte": 1.617821878574846, "learning_rate": 0.0001739222288839694, "loss_nats": 1.1213886737823486, "step": 4040, "unclipped_grad_norm": 0.7582541108131409 }, { "bits_per_byte": 1.6450140555027646, "learning_rate": 0.00017336532827486026, "loss_nats": 1.1402368545532227, "step": 4050, "unclipped_grad_norm": 0.7559582591056824 }, { "bits_per_byte": 1.6581645369183384, "learning_rate": 0.00017280828472537163, "loss_nats": 1.1493520736694336, "step": 4060, "unclipped_grad_norm": 0.743245005607605 }, { "bits_per_byte": 1.680131880913045, "learning_rate": 0.0001722511077538408, "loss_nats": 1.1645786762237549, "step": 4070, "unclipped_grad_norm": 0.7624887824058533 }, { "bits_per_byte": 1.7564567494700543, "learning_rate": 0.00017169380688088497, "loss_nats": 1.2174830436706543, "step": 4080, "unclipped_grad_norm": 0.8132302761077881 }, { "bits_per_byte": 1.6669108866106175, "learning_rate": 0.0001711363916292384, "loss_nats": 1.1554145812988281, "step": 4090, "unclipped_grad_norm": 0.8276432752609253 }, { "bits_per_byte": 1.638365894151012, "learning_rate": 0.00017057887152358974, "loss_nats": 1.1356287002563477, "step": 4100, "unclipped_grad_norm": 0.7751885652542114 }, { "bits_per_byte": 1.6303674970073352, "learning_rate": 0.0001700212560904194, "loss_nats": 1.1300846338272095, "step": 4110, "unclipped_grad_norm": 0.7898787260055542 }, { "bits_per_byte": 1.6896222275529564, "learning_rate": 0.00016946355485783656, "loss_nats": 1.171156883239746, "step": 4120, "unclipped_grad_norm": 0.8148247599601746 }, { "bits_per_byte": 1.6213521664491068, "learning_rate": 0.0001689057773554166, "loss_nats": 1.1238356828689575, "step": 4130, "unclipped_grad_norm": 0.8133904933929443 }, { "bits_per_byte": 1.6881359534842002, "learning_rate": 0.00016834793311403806, "loss_nats": 1.1701266765594482, "step": 4140, "unclipped_grad_norm": 0.8217440247535706 }, { "bits_per_byte": 1.6129334437067215, "learning_rate": 0.00016779003166571981, "loss_nats": 1.1180002689361572, "step": 4150, "unclipped_grad_norm": 0.7947918176651001 }, { "bits_per_byte": 1.651558683298698, "learning_rate": 0.00016723208254345838, "loss_nats": 1.144773244857788, "step": 4160, "unclipped_grad_norm": 0.7860805988311768 }, { "bits_per_byte": 1.6828089628563754, "learning_rate": 0.00016667409528106481, "loss_nats": 1.1664342880249023, "step": 4170, "unclipped_grad_norm": 0.8207855224609375 }, { "bits_per_byte": 1.7311515661874024, "learning_rate": 0.00016611607941300184, "loss_nats": 1.1999428272247314, "step": 4180, "unclipped_grad_norm": 0.8373809456825256 }, { "bits_per_byte": 1.6433404923272186, "learning_rate": 0.0001655580444742211, "loss_nats": 1.139076828956604, "step": 4190, "unclipped_grad_norm": 0.7621071338653564 }, { "bits_per_byte": 1.682679287937625, "learning_rate": 0.00016499999999999997, "loss_nats": 1.166344404220581, "step": 4200, "unclipped_grad_norm": 0.7672834396362305 }, { "bits_per_byte": 1.661693964879345, "learning_rate": 0.0001644419555257789, "loss_nats": 1.1517984867095947, "step": 4210, "unclipped_grad_norm": 0.7330533862113953 }, { "bits_per_byte": 1.6972419188994332, "learning_rate": 0.0001638839205869981, "loss_nats": 1.1764384508132935, "step": 4220, "unclipped_grad_norm": 0.7866858243942261 }, { "bits_per_byte": 1.6670219874030747, "learning_rate": 0.0001633259047189352, "loss_nats": 1.155491590499878, "step": 4230, "unclipped_grad_norm": 0.7902593612670898 }, { "bits_per_byte": 1.6114956687455084, "learning_rate": 0.0001627679174565416, "loss_nats": 1.1170036792755127, "step": 4240, "unclipped_grad_norm": 0.740251362323761 }, { "bits_per_byte": 1.6335233786506678, "learning_rate": 0.00016220996833428016, "loss_nats": 1.1322721242904663, "step": 4250, "unclipped_grad_norm": 0.7777352333068848 }, { "bits_per_byte": 1.6115895712728796, "learning_rate": 0.00016165206688596191, "loss_nats": 1.1170687675476074, "step": 4260, "unclipped_grad_norm": 0.774424135684967 }, { "bits_per_byte": 1.619611874005026, "learning_rate": 0.00016109422264458335, "loss_nats": 1.1226294040679932, "step": 4270, "unclipped_grad_norm": 0.7803477048873901 }, { "bits_per_byte": 1.6299772683725275, "learning_rate": 0.00016053644514216336, "loss_nats": 1.1298141479492188, "step": 4280, "unclipped_grad_norm": 0.7580373287200928 }, { "bits_per_byte": 1.6795242662075467, "learning_rate": 0.00015997874390958058, "loss_nats": 1.164157509803772, "step": 4290, "unclipped_grad_norm": 0.7638103365898132 }, { "bits_per_byte": 1.600082040120998, "learning_rate": 0.00015942112847641018, "loss_nats": 1.109092354774475, "step": 4300, "unclipped_grad_norm": 0.7825598120689392 }, { "bits_per_byte": 1.6771565810531184, "learning_rate": 0.00015886360837076156, "loss_nats": 1.1625163555145264, "step": 4310, "unclipped_grad_norm": 0.7940689921379089 }, { "bits_per_byte": 1.6577068910843924, "learning_rate": 0.00015830619311911495, "loss_nats": 1.149034857749939, "step": 4320, "unclipped_grad_norm": 0.7976187467575073 }, { "bits_per_byte": 1.6195226150092283, "learning_rate": 0.00015774889224615915, "loss_nats": 1.1225675344467163, "step": 4330, "unclipped_grad_norm": 0.7892199158668518 }, { "bits_per_byte": 1.655279183984813, "learning_rate": 0.00015719171527462834, "loss_nats": 1.1473520994186401, "step": 4340, "unclipped_grad_norm": 0.8144169449806213 }, { "bits_per_byte": 1.6394932404274176, "learning_rate": 0.0001566346717251397, "loss_nats": 1.136410117149353, "step": 4350, "unclipped_grad_norm": 0.8119622468948364 }, { "bits_per_byte": 1.6287904160989233, "learning_rate": 0.00015607777111603056, "loss_nats": 1.1289914846420288, "step": 4360, "unclipped_grad_norm": 0.7554351091384888 }, { "bits_per_byte": 1.6321829458698431, "learning_rate": 0.00015552102296319576, "loss_nats": 1.1313430070877075, "step": 4370, "unclipped_grad_norm": 0.808085024356842 }, { "bits_per_byte": 1.6820371047193032, "learning_rate": 0.00015496443677992516, "loss_nats": 1.1658992767333984, "step": 4380, "unclipped_grad_norm": 0.7708722949028015 }, { "bits_per_byte": 1.6522423143358775, "learning_rate": 0.0001544080220767409, "loss_nats": 1.1452471017837524, "step": 4390, "unclipped_grad_norm": 0.8123121857643127 }, { "bits_per_byte": 1.6025224739367414, "learning_rate": 0.00015385178836123511, "loss_nats": 1.1107839345932007, "step": 4400, "unclipped_grad_norm": 0.7717524766921997 }, { "bits_per_byte": 1.6853694405624209, "learning_rate": 0.00015329574513790726, "loss_nats": 1.1682090759277344, "step": 4410, "unclipped_grad_norm": 0.81525057554245 }, { "bits_per_byte": 1.6142167782474597, "learning_rate": 0.00015273990190800186, "loss_nats": 1.1188898086547852, "step": 4420, "unclipped_grad_norm": 0.7956154346466064 }, { "bits_per_byte": 1.6755413199962148, "learning_rate": 0.0001521842681693461, "loss_nats": 1.1613967418670654, "step": 4430, "unclipped_grad_norm": 0.7907513380050659 }, { "bits_per_byte": 1.6011154838700328, "learning_rate": 0.00015162885341618752, "loss_nats": 1.1098086833953857, "step": 4440, "unclipped_grad_norm": 0.7607184648513794 }, { "bits_per_byte": 1.5990663105850023, "learning_rate": 0.00015107366713903178, "loss_nats": 1.1083883047103882, "step": 4450, "unclipped_grad_norm": 0.7442869544029236 }, { "bits_per_byte": 1.5859970049806358, "learning_rate": 0.0001505187188244806, "loss_nats": 1.0993293523788452, "step": 4460, "unclipped_grad_norm": 0.7537500262260437 }, { "bits_per_byte": 1.6451918855637568, "learning_rate": 0.00014996401795506952, "loss_nats": 1.1403601169586182, "step": 4470, "unclipped_grad_norm": 0.7931647896766663 }, { "bits_per_byte": 1.6505943765753102, "learning_rate": 0.00014940957400910585, "loss_nats": 1.1441048383712769, "step": 4480, "unclipped_grad_norm": 0.9106901288032532 }, { "bits_per_byte": 1.6301040195862135, "learning_rate": 0.00014885539646050699, "loss_nats": 1.1299020051956177, "step": 4490, "unclipped_grad_norm": 0.7690432667732239 }, { "bits_per_byte": 1.6520481459230536, "learning_rate": 0.00014830149477863805, "loss_nats": 1.1451125144958496, "step": 4500, "unclipped_grad_norm": 0.7990603446960449 }, { "bits_per_byte": 1.5968599451570833, "learning_rate": 0.00014774787842815066, "loss_nats": 1.1068589687347412, "step": 4510, "unclipped_grad_norm": 0.8130239248275757 }, { "bits_per_byte": 1.6430763269754933, "learning_rate": 0.00014719455686882066, "loss_nats": 1.138893723487854, "step": 4520, "unclipped_grad_norm": 0.7745712399482727 }, { "bits_per_byte": 1.660709880151108, "learning_rate": 0.00014664153955538695, "loss_nats": 1.1511163711547852, "step": 4530, "unclipped_grad_norm": 0.7695059776306152 }, { "bits_per_byte": 1.707295852686222, "learning_rate": 0.00014608883593738946, "loss_nats": 1.1834073066711426, "step": 4540, "unclipped_grad_norm": 0.8436031937599182 }, { "bits_per_byte": 1.5623325361872151, "learning_rate": 0.00014553645545900817, "loss_nats": 1.0829263925552368, "step": 4550, "unclipped_grad_norm": 0.7549967169761658 }, { "bits_per_byte": 1.6257084869954619, "learning_rate": 0.0001449844075589013, "loss_nats": 1.1268552541732788, "step": 4560, "unclipped_grad_norm": 0.8449703454971313 }, { "bits_per_byte": 1.6175644205465043, "learning_rate": 0.0001444327016700444, "loss_nats": 1.1212102174758911, "step": 4570, "unclipped_grad_norm": 0.7528937458992004 }, { "bits_per_byte": 1.570274179056107, "learning_rate": 0.0001438813472195688, "loss_nats": 1.0884311199188232, "step": 4580, "unclipped_grad_norm": 0.790579617023468 }, { "bits_per_byte": 1.609637052237634, "learning_rate": 0.00014333035362860093, "loss_nats": 1.1157153844833374, "step": 4590, "unclipped_grad_norm": 0.7935193181037903 }, { "bits_per_byte": 1.6088013885370918, "learning_rate": 0.00014277973031210088, "loss_nats": 1.1151361465454102, "step": 4600, "unclipped_grad_norm": 0.7915687561035156 }, { "bits_per_byte": 1.6615916352020816, "learning_rate": 0.00014222948667870207, "loss_nats": 1.151727557182312, "step": 4610, "unclipped_grad_norm": 0.8493216633796692 }, { "bits_per_byte": 1.6675298521710726, "learning_rate": 0.00014167963213054991, "loss_nats": 1.1558436155319214, "step": 4620, "unclipped_grad_norm": 0.8179065585136414 }, { "bits_per_byte": 1.6286588493710132, "learning_rate": 0.00014113017606314152, "loss_nats": 1.1289002895355225, "step": 4630, "unclipped_grad_norm": 0.8027536273002625 }, { "bits_per_byte": 1.7031861552612044, "learning_rate": 0.00014058112786516518, "loss_nats": 1.180558681488037, "step": 4640, "unclipped_grad_norm": 0.8179929256439209 }, { "bits_per_byte": 1.6059254946493637, "learning_rate": 0.0001400324969183396, "loss_nats": 1.113142728805542, "step": 4650, "unclipped_grad_norm": 0.8228299021720886 }, { "bits_per_byte": 1.5739184914278899, "learning_rate": 0.00013948429259725406, "loss_nats": 1.0909571647644043, "step": 4660, "unclipped_grad_norm": 0.820624589920044 }, { "bits_per_byte": 1.7237126286069824, "learning_rate": 0.00013893652426920776, "loss_nats": 1.194786548614502, "step": 4670, "unclipped_grad_norm": 0.8582949638366699 }, { "bits_per_byte": 1.7009104808249877, "learning_rate": 0.00013838920129405016, "loss_nats": 1.1789813041687012, "step": 4680, "unclipped_grad_norm": 0.8158925175666809 }, { "bits_per_byte": 1.6679322915740917, "learning_rate": 0.0001378423330240207, "loss_nats": 1.1561225652694702, "step": 4690, "unclipped_grad_norm": 0.849236249923706 }, { "bits_per_byte": 1.589747430648001, "learning_rate": 0.00013729592880358936, "loss_nats": 1.101928949356079, "step": 4700, "unclipped_grad_norm": 0.7403190732002258 }, { "bits_per_byte": 1.6742342518496576, "learning_rate": 0.00013674999796929659, "loss_nats": 1.1604907512664795, "step": 4710, "unclipped_grad_norm": 0.8582251071929932 }, { "bits_per_byte": 1.607032031025015, "learning_rate": 0.00013620454984959417, "loss_nats": 1.1139097213745117, "step": 4720, "unclipped_grad_norm": 0.8400667905807495 }, { "bits_per_byte": 1.5176990826873464, "learning_rate": 0.00013565959376468542, "loss_nats": 1.0519888401031494, "step": 4730, "unclipped_grad_norm": 0.7827285528182983 }, { "bits_per_byte": 1.5868581221135059, "learning_rate": 0.00013511513902636634, "loss_nats": 1.099926233291626, "step": 4740, "unclipped_grad_norm": 0.7815871238708496 }, { "bits_per_byte": 1.6377919880450826, "learning_rate": 0.0001345711949378662, "loss_nats": 1.1352308988571167, "step": 4750, "unclipped_grad_norm": 0.7846289873123169 }, { "bits_per_byte": 1.6125860387519786, "learning_rate": 0.00013402777079368853, "loss_nats": 1.1177594661712646, "step": 4760, "unclipped_grad_norm": 0.8307337164878845 }, { "bits_per_byte": 1.5936545325103024, "learning_rate": 0.0001334848758794528, "loss_nats": 1.1046371459960938, "step": 4770, "unclipped_grad_norm": 0.8213374614715576 }, { "bits_per_byte": 1.6121588338472355, "learning_rate": 0.00013294251947173504, "loss_nats": 1.1174633502960205, "step": 4780, "unclipped_grad_norm": 0.8224621415138245 }, { "bits_per_byte": 1.5705015401205475, "learning_rate": 0.00013240071083790996, "loss_nats": 1.0885887145996094, "step": 4790, "unclipped_grad_norm": 0.8122685551643372 }, { "bits_per_byte": 1.5722774329733569, "learning_rate": 0.0001318594592359921, "loss_nats": 1.0898196697235107, "step": 4800, "unclipped_grad_norm": 0.8455460667610168 }, { "bits_per_byte": 1.6480244398234671, "learning_rate": 0.0001313187739144781, "loss_nats": 1.1423234939575195, "step": 4810, "unclipped_grad_norm": 0.8817458748817444 }, { "bits_per_byte": 1.574972745077679, "learning_rate": 0.0001307786641121882, "loss_nats": 1.0916879177093506, "step": 4820, "unclipped_grad_norm": 0.7954387664794922 }, { "bits_per_byte": 1.608869665649484, "learning_rate": 0.00013023913905810876, "loss_nats": 1.1151834726333618, "step": 4830, "unclipped_grad_norm": 0.8687117695808411 }, { "bits_per_byte": 1.5985519104762718, "learning_rate": 0.00012970020797123432, "loss_nats": 1.1080317497253418, "step": 4840, "unclipped_grad_norm": 0.8505465388298035 }, { "bits_per_byte": 1.6228387844831649, "learning_rate": 0.00012916188006041027, "loss_nats": 1.1248661279678345, "step": 4850, "unclipped_grad_norm": 0.7912812232971191 }, { "bits_per_byte": 1.5432041098103009, "learning_rate": 0.00012862416452417516, "loss_nats": 1.0696675777435303, "step": 4860, "unclipped_grad_norm": 0.7933070063591003 }, { "bits_per_byte": 1.635918581029234, "learning_rate": 0.00012808707055060405, "loss_nats": 1.13393235206604, "step": 4870, "unclipped_grad_norm": 0.8805916905403137 }, { "bits_per_byte": 1.4766798445953597, "learning_rate": 0.000127550607317151, "loss_nats": 1.0235564708709717, "step": 4880, "unclipped_grad_norm": 0.8066675066947937 }, { "bits_per_byte": 1.5344280071194227, "learning_rate": 0.00012701478399049248, "loss_nats": 1.0635844469070435, "step": 4890, "unclipped_grad_norm": 0.8088182806968689 }, { "bits_per_byte": 1.6101516243290155, "learning_rate": 0.0001264796097263708, "loss_nats": 1.1160720586776733, "step": 4900, "unclipped_grad_norm": 0.8653903603553772 }, { "bits_per_byte": 1.6779198400576476, "learning_rate": 0.0001259450936694375, "loss_nats": 1.1630454063415527, "step": 4910, "unclipped_grad_norm": 0.8391038179397583 }, { "bits_per_byte": 1.6284904783558185, "learning_rate": 0.00012541124495309727, "loss_nats": 1.1287835836410522, "step": 4920, "unclipped_grad_norm": 0.8723531365394592 }, { "bits_per_byte": 1.5831391692712484, "learning_rate": 0.00012487807269935167, "loss_nats": 1.0973484516143799, "step": 4930, "unclipped_grad_norm": 0.7553638815879822 }, { "bits_per_byte": 1.5755332365368409, "learning_rate": 0.00012434558601864349, "loss_nats": 1.0920764207839966, "step": 4940, "unclipped_grad_norm": 0.8158645033836365 }, { "bits_per_byte": 1.530142715407874, "learning_rate": 0.0001238137940097009, "loss_nats": 1.0606141090393066, "step": 4950, "unclipped_grad_norm": 0.7773033380508423 }, { "bits_per_byte": 1.5220387209165676, "learning_rate": 0.0001232827057593821, "loss_nats": 1.0549968481063843, "step": 4960, "unclipped_grad_norm": 0.8325244784355164 }, { "bits_per_byte": 1.6031579498316795, "learning_rate": 0.00012275233034251992, "loss_nats": 1.1112244129180908, "step": 4970, "unclipped_grad_norm": 0.8591333627700806 }, { "bits_per_byte": 1.6532000857185325, "learning_rate": 0.0001222226768217669, "loss_nats": 1.1459109783172607, "step": 4980, "unclipped_grad_norm": 0.8351151347160339 }, { "bits_per_byte": 1.585272098107249, "learning_rate": 0.00012169375424744034, "loss_nats": 1.0988268852233887, "step": 4990, "unclipped_grad_norm": 0.8779996037483215 }, { "bits_per_byte": 1.5394700224947675, "learning_rate": 0.00012116557165736772, "loss_nats": 1.0670793056488037, "step": 5000, "unclipped_grad_norm": 0.7928086519241333 }, { "bits_per_byte": 1.5552827953456976, "learning_rate": 0.00012063813807673214, "loss_nats": 1.0780398845672607, "step": 5010, "unclipped_grad_norm": 0.8201308846473694 }, { "bits_per_byte": 1.5898205232746179, "learning_rate": 0.00012011146251791832, "loss_nats": 1.1019796133041382, "step": 5020, "unclipped_grad_norm": 0.7774789929389954 }, { "bits_per_byte": 1.5697566832596608, "learning_rate": 0.0001195855539803584, "loss_nats": 1.088072419166565, "step": 5030, "unclipped_grad_norm": 0.7855319380760193 }, { "bits_per_byte": 1.539816567536256, "learning_rate": 0.00011906042145037817, "loss_nats": 1.0673195123672485, "step": 5040, "unclipped_grad_norm": 0.7920348048210144 }, { "bits_per_byte": 1.5092143186070297, "learning_rate": 0.0001185360739010438, "loss_nats": 1.0461076498031616, "step": 5050, "unclipped_grad_norm": 0.8045132160186768 }, { "bits_per_byte": 1.5339026001210367, "learning_rate": 0.00011801252029200808, "loss_nats": 1.0632202625274658, "step": 5060, "unclipped_grad_norm": 0.8044583797454834 }, { "bits_per_byte": 1.6138325690054323, "learning_rate": 0.00011748976956935777, "loss_nats": 1.1186234951019287, "step": 5070, "unclipped_grad_norm": 0.8723475337028503 }, { "bits_per_byte": 1.574335549356232, "learning_rate": 0.00011696783066546024, "loss_nats": 1.091246247291565, "step": 5080, "unclipped_grad_norm": 0.8741123080253601 }, { "bits_per_byte": 1.5594206979254568, "learning_rate": 0.00011644671249881147, "loss_nats": 1.0809080600738525, "step": 5090, "unclipped_grad_norm": 0.8465037941932678 }, { "bits_per_byte": 1.5640659493252613, "learning_rate": 0.00011592642397388293, "loss_nats": 1.0841279029846191, "step": 5100, "unclipped_grad_norm": 0.783299446105957 }, { "bits_per_byte": 1.488738924108558, "learning_rate": 0.00011540697398097015, "loss_nats": 1.0319151878356934, "step": 5110, "unclipped_grad_norm": 0.8348572254180908 }, { "bits_per_byte": 1.5847128105266433, "learning_rate": 0.00011488837139604016, "loss_nats": 1.0984392166137695, "step": 5120, "unclipped_grad_norm": 0.8110998868942261 }, { "bits_per_byte": 1.590178763136365, "learning_rate": 0.00011437062508058039, "loss_nats": 1.1022279262542725, "step": 5130, "unclipped_grad_norm": 0.8208310604095459 }, { "bits_per_byte": 1.5494388248693793, "learning_rate": 0.00011385374388144673, "loss_nats": 1.0739891529083252, "step": 5140, "unclipped_grad_norm": 0.8678956627845764 }, { "bits_per_byte": 1.6273239200350162, "learning_rate": 0.00011333773663071286, "loss_nats": 1.1279749870300293, "step": 5150, "unclipped_grad_norm": 0.848126232624054 }, { "bits_per_byte": 1.5085432423033631, "learning_rate": 0.00011282261214551897, "loss_nats": 1.0456424951553345, "step": 5160, "unclipped_grad_norm": 0.7922812104225159 }, { "bits_per_byte": 1.6700107019097687, "learning_rate": 0.00011230837922792112, "loss_nats": 1.1575632095336914, "step": 5170, "unclipped_grad_norm": 0.8759046196937561 }, { "bits_per_byte": 1.5869253673299932, "learning_rate": 0.0001117950466647412, "loss_nats": 1.0999728441238403, "step": 5180, "unclipped_grad_norm": 0.8647724390029907 }, { "bits_per_byte": 1.5237559676854329, "learning_rate": 0.00011128262322741627, "loss_nats": 1.0561871528625488, "step": 5190, "unclipped_grad_norm": 0.813337504863739 }, { "bits_per_byte": 1.556985939537192, "learning_rate": 0.00011077111767184913, "loss_nats": 1.0792204141616821, "step": 5200, "unclipped_grad_norm": 0.8388134241104126 }, { "bits_per_byte": 1.5602121620847276, "learning_rate": 0.00011026053873825837, "loss_nats": 1.0814566612243652, "step": 5210, "unclipped_grad_norm": 0.8162683248519897 }, { "bits_per_byte": 1.6432956048553435, "learning_rate": 0.0001097508951510293, "loss_nats": 1.1390457153320312, "step": 5220, "unclipped_grad_norm": 0.8667425513267517 }, { "bits_per_byte": 1.5501931407760636, "learning_rate": 0.0001092421956185646, "loss_nats": 1.074512004852295, "step": 5230, "unclipped_grad_norm": 0.8668702840805054 }, { "bits_per_byte": 1.5510165937083948, "learning_rate": 0.00010873444883313581, "loss_nats": 1.075082778930664, "step": 5240, "unclipped_grad_norm": 0.8049055337905884 }, { "bits_per_byte": 1.5393878147876552, "learning_rate": 0.00010822766347073447, "loss_nats": 1.0670223236083984, "step": 5250, "unclipped_grad_norm": 0.8209428191184998 }, { "bits_per_byte": 1.540701762240247, "learning_rate": 0.00010772184819092424, "loss_nats": 1.0679330825805664, "step": 5260, "unclipped_grad_norm": 0.8524174690246582 }, { "bits_per_byte": 1.5376201771020872, "learning_rate": 0.0001072170116366926, "loss_nats": 1.0657970905303955, "step": 5270, "unclipped_grad_norm": 0.8064137697219849 }, { "bits_per_byte": 1.5550475230793173, "learning_rate": 0.00010671316243430345, "loss_nats": 1.0778768062591553, "step": 5280, "unclipped_grad_norm": 0.8169121146202087 }, { "bits_per_byte": 1.5423493560355128, "learning_rate": 0.00010621030919314944, "loss_nats": 1.069075107574463, "step": 5290, "unclipped_grad_norm": 0.8263237476348877 }, { "bits_per_byte": 1.5542627662434303, "learning_rate": 0.000105708460505605, "loss_nats": 1.077332854270935, "step": 5300, "unclipped_grad_norm": 0.8492859601974487 }, { "bits_per_byte": 1.5509104804128124, "learning_rate": 0.00010520762494687964, "loss_nats": 1.0750092267990112, "step": 5310, "unclipped_grad_norm": 0.8885292410850525 }, { "bits_per_byte": 1.5517654061702515, "learning_rate": 0.00010470781107487111, "loss_nats": 1.0756018161773682, "step": 5320, "unclipped_grad_norm": 0.8326203227043152 }, { "bits_per_byte": 1.5477100552629064, "learning_rate": 0.00010420902743001955, "loss_nats": 1.0727908611297607, "step": 5330, "unclipped_grad_norm": 0.8806997537612915 }, { "bits_per_byte": 1.503639328996661, "learning_rate": 0.00010371128253516117, "loss_nats": 1.0422433614730835, "step": 5340, "unclipped_grad_norm": 0.8234722018241882 }, { "bits_per_byte": 1.5869049013945407, "learning_rate": 0.000103214584895383, "loss_nats": 1.0999586582183838, "step": 5350, "unclipped_grad_norm": 0.8706242442131042 }, { "bits_per_byte": 1.5072631754329913, "learning_rate": 0.00010271894299787715, "loss_nats": 1.044755220413208, "step": 5360, "unclipped_grad_norm": 0.8708282709121704 }, { "bits_per_byte": 1.471025742965594, "learning_rate": 0.00010222436531179622, "loss_nats": 1.0196373462677002, "step": 5370, "unclipped_grad_norm": 0.8303158283233643 }, { "bits_per_byte": 1.5968417149960918, "learning_rate": 0.0001017308602881082, "loss_nats": 1.1068463325500488, "step": 5380, "unclipped_grad_norm": 0.883955180644989 }, { "bits_per_byte": 1.531507741707772, "learning_rate": 0.00010123843635945242, "loss_nats": 1.0615602731704712, "step": 5390, "unclipped_grad_norm": 0.8507125377655029 }, { "bits_per_byte": 1.5483677169198058, "learning_rate": 0.00010074710193999505, "loss_nats": 1.073246717453003, "step": 5400, "unclipped_grad_norm": 0.8522453904151917 }, { "bits_per_byte": 1.571479089508052, "learning_rate": 0.0001002568654252858, "loss_nats": 1.089266300201416, "step": 5410, "unclipped_grad_norm": 0.8313266038894653 }, { "bits_per_byte": 1.5408727129952047, "learning_rate": 9.976773519211404e-05, "loss_nats": 1.0680515766143799, "step": 5420, "unclipped_grad_norm": 0.8902517557144165 }, { "bits_per_byte": 1.5778336764747818, "learning_rate": 9.92797195983659e-05, "loss_nats": 1.0936709642410278, "step": 5430, "unclipped_grad_norm": 0.8956695795059204 }, { "bits_per_byte": 1.513927503153925, "learning_rate": 9.879282698288142e-05, "loss_nats": 1.0493745803833008, "step": 5440, "unclipped_grad_norm": 0.897573709487915 }, { "bits_per_byte": 1.5078418970531444, "learning_rate": 9.830706566531197e-05, "loss_nats": 1.0451563596725464, "step": 5450, "unclipped_grad_norm": 0.8440007567405701 }, { "bits_per_byte": 1.5030907043404085, "learning_rate": 9.782244394597826e-05, "loss_nats": 1.0418630838394165, "step": 5460, "unclipped_grad_norm": 0.8436890244483948 }, { "bits_per_byte": 1.5464133060754008, "learning_rate": 9.733897010572829e-05, "loss_nats": 1.0718920230865479, "step": 5470, "unclipped_grad_norm": 0.8681005239486694 }, { "bits_per_byte": 1.553998600891705, "learning_rate": 9.685665240579612e-05, "loss_nats": 1.077149748802185, "step": 5480, "unclipped_grad_norm": 0.8523774147033691 }, { "bits_per_byte": 1.543775436176467, "learning_rate": 9.637549908766041e-05, "loss_nats": 1.070063591003418, "step": 5490, "unclipped_grad_norm": 0.8906207084655762 }, { "bits_per_byte": 1.5444891641775478, "learning_rate": 9.589551837290389e-05, "loss_nats": 1.0705583095550537, "step": 5500, "unclipped_grad_norm": 0.8658390045166016 }, { "bits_per_byte": 1.6370650173798855, "learning_rate": 9.541671846307257e-05, "loss_nats": 1.1347270011901855, "step": 5510, "unclipped_grad_norm": 0.9161565899848938 }, { "bits_per_byte": 1.5520985365649729, "learning_rate": 9.493910753953597e-05, "loss_nats": 1.075832724571228, "step": 5520, "unclipped_grad_norm": 0.845907986164093 }, { "bits_per_byte": 1.5215033389244317, "learning_rate": 9.446269376334689e-05, "loss_nats": 1.0546257495880127, "step": 5530, "unclipped_grad_norm": 0.8585748672485352 }, { "bits_per_byte": 1.5943249208833656, "learning_rate": 9.39874852751024e-05, "loss_nats": 1.1051018238067627, "step": 5540, "unclipped_grad_norm": 0.9360165596008301 }, { "bits_per_byte": 1.5176287417831436, "learning_rate": 9.351349019480433e-05, "loss_nats": 1.0519400835037231, "step": 5550, "unclipped_grad_norm": 0.8308430910110474 }, { "bits_per_byte": 1.5164790377621256, "learning_rate": 9.304071662172086e-05, "loss_nats": 1.0511431694030762, "step": 5560, "unclipped_grad_norm": 0.894058346748352 }, { "bits_per_byte": 1.4905304673825959, "learning_rate": 9.256917263424787e-05, "loss_nats": 1.0331569910049438, "step": 5570, "unclipped_grad_norm": 0.8399901390075684 }, { "bits_per_byte": 1.5655759569998369, "learning_rate": 9.209886628977114e-05, "loss_nats": 1.085174560546875, "step": 5580, "unclipped_grad_norm": 0.8579773902893066 }, { "bits_per_byte": 1.482571282283316, "learning_rate": 9.162980562452843e-05, "loss_nats": 1.0276401042938232, "step": 5590, "unclipped_grad_norm": 0.8161733746528625 }, { "bits_per_byte": 1.5496305855250914, "learning_rate": 9.116199865347238e-05, "loss_nats": 1.0741220712661743, "step": 5600, "unclipped_grad_norm": 0.8440633416175842 }, { "bits_per_byte": 1.5466313800766949, "learning_rate": 9.069545337013333e-05, "loss_nats": 1.0720431804656982, "step": 5610, "unclipped_grad_norm": 0.8384022116661072 }, { "bits_per_byte": 1.55366271877457, "learning_rate": 9.0230177746483e-05, "loss_nats": 1.0769169330596924, "step": 5620, "unclipped_grad_norm": 0.8643615245819092 }, { "bits_per_byte": 1.5359743031333302, "learning_rate": 8.976617973279797e-05, "loss_nats": 1.0646562576293945, "step": 5630, "unclipped_grad_norm": 0.8559057712554932 }, { "bits_per_byte": 1.5154779267514533, "learning_rate": 8.930346725752419e-05, "loss_nats": 1.050449252128601, "step": 5640, "unclipped_grad_norm": 0.8564271330833435 }, { "bits_per_byte": 1.5399061704973558, "learning_rate": 8.88420482271411e-05, "loss_nats": 1.0673816204071045, "step": 5650, "unclipped_grad_norm": 0.899230420589447 }, { "bits_per_byte": 1.6017691899259623, "learning_rate": 8.838193052602691e-05, "loss_nats": 1.1102617979049683, "step": 5660, "unclipped_grad_norm": 0.8796955943107605 }, { "bits_per_byte": 1.5203328250026595, "learning_rate": 8.792312201632352e-05, "loss_nats": 1.05381441116333, "step": 5670, "unclipped_grad_norm": 0.8509314060211182 }, { "bits_per_byte": 1.5301421994599214, "learning_rate": 8.746563053780261e-05, "loss_nats": 1.060613751411438, "step": 5680, "unclipped_grad_norm": 0.8624072074890137 }, { "bits_per_byte": 1.6028342784827556, "learning_rate": 8.700946390773114e-05, "loss_nats": 1.1110000610351562, "step": 5690, "unclipped_grad_norm": 0.8914096355438232 }, { "bits_per_byte": 1.526354281574668, "learning_rate": 8.655462992073833e-05, "loss_nats": 1.057988166809082, "step": 5700, "unclipped_grad_norm": 0.8579604625701904 }, { "bits_per_byte": 1.5347007716036911, "learning_rate": 8.610113634868199e-05, "loss_nats": 1.063773512840271, "step": 5710, "unclipped_grad_norm": 0.8810272216796875 }, { "bits_per_byte": 1.5491378552303696, "learning_rate": 8.564899094051613e-05, "loss_nats": 1.0737805366516113, "step": 5720, "unclipped_grad_norm": 0.8394830226898193 }, { "bits_per_byte": 1.5456280332915613, "learning_rate": 8.519820142215816e-05, "loss_nats": 1.071347713470459, "step": 5730, "unclipped_grad_norm": 0.865943968296051 }, { "bits_per_byte": 1.5433466834278662, "learning_rate": 8.474877549635728e-05, "loss_nats": 1.0697664022445679, "step": 5740, "unclipped_grad_norm": 0.886944591999054 }, { "bits_per_byte": 1.5488365416260579, "learning_rate": 8.430072084256247e-05, "loss_nats": 1.0735716819763184, "step": 5750, "unclipped_grad_norm": 0.9145897626876831 }, { "bits_per_byte": 1.5636783004302166, "learning_rate": 8.385404511679161e-05, "loss_nats": 1.0838592052459717, "step": 5760, "unclipped_grad_norm": 0.9009037017822266 }, { "bits_per_byte": 1.511110599315443, "learning_rate": 8.340875595150034e-05, "loss_nats": 1.0474220514297485, "step": 5770, "unclipped_grad_norm": 0.844070315361023 }, { "bits_per_byte": 1.494012772097266, "learning_rate": 8.296486095545203e-05, "loss_nats": 1.035570740699768, "step": 5780, "unclipped_grad_norm": 0.8943845629692078 }, { "bits_per_byte": 1.5135893852623288, "learning_rate": 8.252236771358729e-05, "loss_nats": 1.049140214920044, "step": 5790, "unclipped_grad_norm": 0.8841454386711121 }, { "bits_per_byte": 1.4905120652389536, "learning_rate": 8.208128378689484e-05, "loss_nats": 1.033144235610962, "step": 5800, "unclipped_grad_norm": 0.8580894470214844 }, { "bits_per_byte": 1.5735937881830608, "learning_rate": 8.16416167122819e-05, "loss_nats": 1.0907320976257324, "step": 5810, "unclipped_grad_norm": 0.933175265789032 }, { "bits_per_byte": 1.5042643139498963, "learning_rate": 8.120337400244578e-05, "loss_nats": 1.042676568031311, "step": 5820, "unclipped_grad_norm": 0.828717052936554 }, { "bits_per_byte": 1.5156406223391694, "learning_rate": 8.076656314574521e-05, "loss_nats": 1.0505620241165161, "step": 5830, "unclipped_grad_norm": 0.9511673450469971 }, { "bits_per_byte": 1.5092205099824607, "learning_rate": 8.03311916060725e-05, "loss_nats": 1.0461119413375854, "step": 5840, "unclipped_grad_norm": 0.8395604491233826 }, { "bits_per_byte": 1.477936693807865, "learning_rate": 7.989726682272606e-05, "loss_nats": 1.0244276523590088, "step": 5850, "unclipped_grad_norm": 0.9143134355545044 }, { "bits_per_byte": 1.5336375748560571, "learning_rate": 7.946479621028318e-05, "loss_nats": 1.063036561012268, "step": 5860, "unclipped_grad_norm": 0.9014923572540283 }, { "bits_per_byte": 1.5402322496033916, "learning_rate": 7.903378715847343e-05, "loss_nats": 1.0676076412200928, "step": 5870, "unclipped_grad_norm": 0.8728002905845642 }, { "bits_per_byte": 1.555285031120159, "learning_rate": 7.860424703205224e-05, "loss_nats": 1.078041434288025, "step": 5880, "unclipped_grad_norm": 0.8936865925788879 }, { "bits_per_byte": 1.564054942435606, "learning_rate": 7.817618317067534e-05, "loss_nats": 1.084120273590088, "step": 5890, "unclipped_grad_norm": 0.8752698302268982 }, { "bits_per_byte": 1.5458715607251832, "learning_rate": 7.774960288877295e-05, "loss_nats": 1.071516513824463, "step": 5900, "unclipped_grad_norm": 0.8957662582397461 }, { "bits_per_byte": 1.5568008862048637, "learning_rate": 7.732451347542522e-05, "loss_nats": 1.0790921449661255, "step": 5910, "unclipped_grad_norm": 0.8808028101921082 }, { "bits_per_byte": 1.4896764015384112, "learning_rate": 7.690092219423722e-05, "loss_nats": 1.0325649976730347, "step": 5920, "unclipped_grad_norm": 0.8660398721694946 }, { "bits_per_byte": 1.5908711652887386, "learning_rate": 7.647883628321536e-05, "loss_nats": 1.102707862854004, "step": 5930, "unclipped_grad_norm": 0.9604227542877197 }, { "bits_per_byte": 1.4574508083676907, "learning_rate": 7.605826295464316e-05, "loss_nats": 1.010227918624878, "step": 5940, "unclipped_grad_norm": 0.8380697965621948 }, { "bits_per_byte": 1.5131755950043528, "learning_rate": 7.563920939495851e-05, "loss_nats": 1.0488533973693848, "step": 5950, "unclipped_grad_norm": 0.8392192125320435 }, { "bits_per_byte": 1.5452194025131114, "learning_rate": 7.522168276463047e-05, "loss_nats": 1.0710644721984863, "step": 5960, "unclipped_grad_norm": 0.932554304599762 }, { "bits_per_byte": 1.5213482105733536, "learning_rate": 7.480569019803719e-05, "loss_nats": 1.054518222808838, "step": 5970, "unclipped_grad_norm": 0.927780270576477 }, { "bits_per_byte": 1.5601101763727658, "learning_rate": 7.439123880334388e-05, "loss_nats": 1.0813859701156616, "step": 5980, "unclipped_grad_norm": 0.8920019268989563 }, { "bits_per_byte": 1.5382338112003657, "learning_rate": 7.39783356623813e-05, "loss_nats": 1.0662224292755127, "step": 5990, "unclipped_grad_norm": 0.9537093639373779 }, { "bits_per_byte": 1.4675097296513546, "learning_rate": 7.356698783052497e-05, "loss_nats": 1.017200231552124, "step": 6000, "unclipped_grad_norm": 0.8773758411407471 }, { "bits_per_byte": 1.506333093257125, "learning_rate": 7.315720233657431e-05, "loss_nats": 1.0441105365753174, "step": 6010, "unclipped_grad_norm": 0.8785988092422485 }, { "bits_per_byte": 1.4794742187065786, "learning_rate": 7.274898618263286e-05, "loss_nats": 1.0254933834075928, "step": 6020, "unclipped_grad_norm": 0.8875186443328857 }, { "bits_per_byte": 1.5245675538148544, "learning_rate": 7.234234634398831e-05, "loss_nats": 1.056749701499939, "step": 6030, "unclipped_grad_norm": 0.88789963722229 }, { "bits_per_byte": 1.5628784091210537, "learning_rate": 7.193728976899362e-05, "loss_nats": 1.083304762840271, "step": 6040, "unclipped_grad_norm": 0.9149079322814941 }, { "bits_per_byte": 1.4881958028971334, "learning_rate": 7.153382337894797e-05, "loss_nats": 1.031538724899292, "step": 6050, "unclipped_grad_norm": 0.9299342632293701 }, { "bits_per_byte": 1.4785307218839452, "learning_rate": 7.113195406797887e-05, "loss_nats": 1.0248394012451172, "step": 6060, "unclipped_grad_norm": 0.8783867359161377 }, { "bits_per_byte": 1.5509557118499895, "learning_rate": 7.073168870292397e-05, "loss_nats": 1.075040578842163, "step": 6070, "unclipped_grad_norm": 0.9337034821510315 }, { "bits_per_byte": 1.4914187577743023, "learning_rate": 7.033303412321404e-05, "loss_nats": 1.0337727069854736, "step": 6080, "unclipped_grad_norm": 0.9420101046562195 }, { "bits_per_byte": 1.4968499698885498, "learning_rate": 6.993599714075586e-05, "loss_nats": 1.0375373363494873, "step": 6090, "unclipped_grad_norm": 0.8664546012878418 }, { "bits_per_byte": 1.4906561867003767, "learning_rate": 6.954058453981609e-05, "loss_nats": 1.0332441329956055, "step": 6100, "unclipped_grad_norm": 0.8687640428543091 }, { "bits_per_byte": 1.5538661742505406, "learning_rate": 6.914680307690504e-05, "loss_nats": 1.077057957649231, "step": 6110, "unclipped_grad_norm": 0.8986173868179321 }, { "bits_per_byte": 1.5476213122150613, "learning_rate": 6.875465948066139e-05, "loss_nats": 1.0727293491363525, "step": 6120, "unclipped_grad_norm": 0.9189077019691467 }, { "bits_per_byte": 1.4789527673091625, "learning_rate": 6.836416045173733e-05, "loss_nats": 1.0251319408416748, "step": 6130, "unclipped_grad_norm": 0.8851786851882935 }, { "bits_per_byte": 1.4768556108645414, "learning_rate": 6.797531266268377e-05, "loss_nats": 1.0236783027648926, "step": 6140, "unclipped_grad_norm": 0.9043483138084412 }, { "bits_per_byte": 1.552988202817886, "learning_rate": 6.758812275783663e-05, "loss_nats": 1.0764493942260742, "step": 6150, "unclipped_grad_norm": 0.9005390405654907 }, { "bits_per_byte": 1.3994871515820966, "learning_rate": 6.720259735320298e-05, "loss_nats": 0.970050573348999, "step": 6160, "unclipped_grad_norm": 0.8554620742797852 }, { "bits_per_byte": 1.5302097886417105, "learning_rate": 6.681874303634842e-05, "loss_nats": 1.0606606006622314, "step": 6170, "unclipped_grad_norm": 0.8954014778137207 }, { "bits_per_byte": 1.4938230752333643, "learning_rate": 6.643656636628402e-05, "loss_nats": 1.0354392528533936, "step": 6180, "unclipped_grad_norm": 0.892278790473938 }, { "bits_per_byte": 1.516749394489282, "learning_rate": 6.605607387335466e-05, "loss_nats": 1.05133056640625, "step": 6190, "unclipped_grad_norm": 0.9185703992843628 }, { "bits_per_byte": 1.5086576107661869, "learning_rate": 6.567727205912724e-05, "loss_nats": 1.0457217693328857, "step": 6200, "unclipped_grad_norm": 0.8927735090255737 }, { "bits_per_byte": 1.5088584865023946, "learning_rate": 6.530016739627959e-05, "loss_nats": 1.045861005783081, "step": 6210, "unclipped_grad_norm": 0.8906724452972412 }, { "bits_per_byte": 1.5118191678369979, "learning_rate": 6.492476632848993e-05, "loss_nats": 1.0479131937026978, "step": 6220, "unclipped_grad_norm": 0.9077256917953491 }, { "bits_per_byte": 1.4471287536282067, "learning_rate": 6.455107527032678e-05, "loss_nats": 1.0030732154846191, "step": 6230, "unclipped_grad_norm": 0.8839235305786133 }, { "bits_per_byte": 1.5076786855174757, "learning_rate": 6.417910060713926e-05, "loss_nats": 1.0450432300567627, "step": 6240, "unclipped_grad_norm": 0.9515005946159363 }, { "bits_per_byte": 1.5408861276419719, "learning_rate": 6.380884869494799e-05, "loss_nats": 1.0680608749389648, "step": 6250, "unclipped_grad_norm": 0.901561975479126 }, { "bits_per_byte": 1.4200397662908057, "learning_rate": 6.34403258603367e-05, "loss_nats": 0.9842965602874756, "step": 6260, "unclipped_grad_norm": 0.889621376991272 }, { "bits_per_byte": 1.5216213190229235, "learning_rate": 6.30735384003437e-05, "loss_nats": 1.0547075271606445, "step": 6270, "unclipped_grad_norm": 0.9411186575889587 }, { "bits_per_byte": 1.5224643779774532, "learning_rate": 6.270849258235484e-05, "loss_nats": 1.0552918910980225, "step": 6280, "unclipped_grad_norm": 0.9406285881996155 }, { "bits_per_byte": 1.452789390598705, "learning_rate": 6.234519464399581e-05, "loss_nats": 1.0069968700408936, "step": 6290, "unclipped_grad_norm": 0.8839805126190186 }, { "bits_per_byte": 1.5074908804627336, "learning_rate": 6.198365079302614e-05, "loss_nats": 1.0449130535125732, "step": 6300, "unclipped_grad_norm": 0.9184122681617737 }, { "bits_per_byte": 1.478450577968643, "learning_rate": 6.16238672072326e-05, "loss_nats": 1.0247838497161865, "step": 6310, "unclipped_grad_norm": 0.8948619365692139 }, { "bits_per_byte": 1.4214500240278807, "learning_rate": 6.126585003432406e-05, "loss_nats": 0.985274076461792, "step": 6320, "unclipped_grad_norm": 0.8749227523803711 }, { "bits_per_byte": 1.5185149683830395, "learning_rate": 6.090960539182614e-05, "loss_nats": 1.0525543689727783, "step": 6330, "unclipped_grad_norm": 0.9816636443138123 }, { "bits_per_byte": 1.4759477144506365, "learning_rate": 6.0555139366976935e-05, "loss_nats": 1.023048996925354, "step": 6340, "unclipped_grad_norm": 0.8950188159942627 }, { "bits_per_byte": 1.587707716408768, "learning_rate": 6.020245801662274e-05, "loss_nats": 1.1005151271820068, "step": 6350, "unclipped_grad_norm": 1.01009202003479 }, { "bits_per_byte": 1.4638088349874379, "learning_rate": 5.985156736711483e-05, "loss_nats": 1.0146349668502808, "step": 6360, "unclipped_grad_norm": 0.9816907644271851 }, { "bits_per_byte": 1.4382974445064014, "learning_rate": 5.950247341420618e-05, "loss_nats": 0.9969518184661865, "step": 6370, "unclipped_grad_norm": 0.8970034122467041 }, { "bits_per_byte": 1.4818757844432269, "learning_rate": 5.915518212294935e-05, "loss_nats": 1.0271580219268799, "step": 6380, "unclipped_grad_norm": 0.9058489799499512 }, { "bits_per_byte": 1.527050123380059, "learning_rate": 5.880969942759425e-05, "loss_nats": 1.0584704875946045, "step": 6390, "unclipped_grad_norm": 0.929339587688446 }, { "bits_per_byte": 1.5156593684481134, "learning_rate": 5.846603123148688e-05, "loss_nats": 1.0505750179290771, "step": 6400, "unclipped_grad_norm": 0.9009133577346802 }, { "bits_per_byte": 1.4969940913499729, "learning_rate": 5.812418340696858e-05, "loss_nats": 1.0376372337341309, "step": 6410, "unclipped_grad_norm": 0.957128643989563 }, { "bits_per_byte": 1.4783625228514012, "learning_rate": 5.778416179527541e-05, "loss_nats": 1.0247228145599365, "step": 6420, "unclipped_grad_norm": 0.9414587616920471 }, { "bits_per_byte": 1.523227980947284, "learning_rate": 5.7445972206438635e-05, "loss_nats": 1.055821180343628, "step": 6430, "unclipped_grad_norm": 0.9234302639961243 }, { "bits_per_byte": 1.4623229048839832, "learning_rate": 5.710962041918516e-05, "loss_nats": 1.013604998588562, "step": 6440, "unclipped_grad_norm": 0.9177728295326233 }, { "bits_per_byte": 1.5306256426914968, "learning_rate": 5.677511218083908e-05, "loss_nats": 1.0609488487243652, "step": 6450, "unclipped_grad_norm": 0.9453822374343872 }, { "bits_per_byte": 1.5847898587542297, "learning_rate": 5.644245320722316e-05, "loss_nats": 1.0984926223754883, "step": 6460, "unclipped_grad_norm": 0.9647909998893738 }, { "bits_per_byte": 1.5690988496201106, "learning_rate": 5.61116491825615e-05, "loss_nats": 1.0876164436340332, "step": 6470, "unclipped_grad_norm": 0.9592546820640564 }, { "bits_per_byte": 1.4544314809491432, "learning_rate": 5.5782705759382104e-05, "loss_nats": 1.0081350803375244, "step": 6480, "unclipped_grad_norm": 0.9334172010421753 }, { "bits_per_byte": 1.5573938823850386, "learning_rate": 5.5455628558420546e-05, "loss_nats": 1.0795031785964966, "step": 6490, "unclipped_grad_norm": 0.967822790145874 }, { "bits_per_byte": 1.5280063469188563, "learning_rate": 5.513042316852368e-05, "loss_nats": 1.0591332912445068, "step": 6500, "unclipped_grad_norm": 0.9625112414360046 }, { "bits_per_byte": 1.4687796495453256, "learning_rate": 5.480709514655443e-05, "loss_nats": 1.018080472946167, "step": 6510, "unclipped_grad_norm": 0.9510544538497925 }, { "bits_per_byte": 1.4284500618832987, "learning_rate": 5.448565001729653e-05, "loss_nats": 0.9901261329650879, "step": 6520, "unclipped_grad_norm": 0.8985839486122131 }, { "bits_per_byte": 1.4546447394362132, "learning_rate": 5.41660932733604e-05, "loss_nats": 1.0082828998565674, "step": 6530, "unclipped_grad_norm": 0.907886803150177 }, { "bits_per_byte": 1.4970680438898438, "learning_rate": 5.384843037508902e-05, "loss_nats": 1.0376884937286377, "step": 6540, "unclipped_grad_norm": 0.9381445050239563 }, { "bits_per_byte": 1.4280130539674563, "learning_rate": 5.353266675046498e-05, "loss_nats": 0.9898232221603394, "step": 6550, "unclipped_grad_norm": 0.9031082391738892 }, { "bits_per_byte": 1.5210671909218436, "learning_rate": 5.321880779501729e-05, "loss_nats": 1.054323434829712, "step": 6560, "unclipped_grad_norm": 0.9433532357215881 }, { "bits_per_byte": 1.479243074023819, "learning_rate": 5.290685887172962e-05, "loss_nats": 1.0253331661224365, "step": 6570, "unclipped_grad_norm": 0.9311387538909912 }, { "bits_per_byte": 1.5051876888023787, "learning_rate": 5.2596825310948295e-05, "loss_nats": 1.0433166027069092, "step": 6580, "unclipped_grad_norm": 0.9569925665855408 }, { "bits_per_byte": 1.494823842278735, "learning_rate": 5.228871241029154e-05, "loss_nats": 1.0361329317092896, "step": 6590, "unclipped_grad_norm": 0.8958868384361267 }, { "bits_per_byte": 1.4302109062541577, "learning_rate": 5.198252543455865e-05, "loss_nats": 0.9913466572761536, "step": 6600, "unclipped_grad_norm": 0.9136191010475159 }, { "bits_per_byte": 1.436879103584736, "learning_rate": 5.167826961564027e-05, "loss_nats": 0.9959686994552612, "step": 6610, "unclipped_grad_norm": 0.8710504770278931 }, { "bits_per_byte": 1.4278971376607748, "learning_rate": 5.137595015242884e-05, "loss_nats": 0.9897428750991821, "step": 6620, "unclipped_grad_norm": 0.9145034551620483 }, { "bits_per_byte": 1.4582168190946336, "learning_rate": 5.1075572210729925e-05, "loss_nats": 1.010758876800537, "step": 6630, "unclipped_grad_norm": 0.8999127149581909 }, { "bits_per_byte": 1.5240701799885592, "learning_rate": 5.07771409231737e-05, "loss_nats": 1.056404948234558, "step": 6640, "unclipped_grad_norm": 0.9801992774009705 }, { "bits_per_byte": 1.5169175935218258, "learning_rate": 5.048066138912756e-05, "loss_nats": 1.0514471530914307, "step": 6650, "unclipped_grad_norm": 0.8882671594619751 }, { "bits_per_byte": 1.5152221885496202, "learning_rate": 5.018613867460863e-05, "loss_nats": 1.050271987915039, "step": 6660, "unclipped_grad_norm": 0.9029646515846252 }, { "bits_per_byte": 1.4896082964086694, "learning_rate": 4.989357781219755e-05, "loss_nats": 1.0325177907943726, "step": 6670, "unclipped_grad_norm": 0.9797500371932983 }, { "bits_per_byte": 1.4764526555135697, "learning_rate": 4.960298380095222e-05, "loss_nats": 1.023398995399475, "step": 6680, "unclipped_grad_norm": 0.9356769919395447 }, { "bits_per_byte": 1.4958203097578342, "learning_rate": 4.931436160632256e-05, "loss_nats": 1.0368236303329468, "step": 6690, "unclipped_grad_norm": 0.939789891242981 }, { "bits_per_byte": 1.5480882451121538, "learning_rate": 4.9027716160065485e-05, "loss_nats": 1.073053002357483, "step": 6700, "unclipped_grad_norm": 1.0080333948135376 }, { "bits_per_byte": 1.525887864625528, "learning_rate": 4.874305236016084e-05, "loss_nats": 1.0576648712158203, "step": 6710, "unclipped_grad_norm": 0.9197862148284912 }, { "bits_per_byte": 1.534906462854123, "learning_rate": 4.846037507072752e-05, "loss_nats": 1.0639160871505737, "step": 6720, "unclipped_grad_norm": 0.9543269872665405 }, { "bits_per_byte": 1.4820493149379474, "learning_rate": 4.817968912194049e-05, "loss_nats": 1.0272783041000366, "step": 6730, "unclipped_grad_norm": 0.9817740321159363 }, { "bits_per_byte": 1.4384499931177168, "learning_rate": 4.790099930994813e-05, "loss_nats": 0.9970575571060181, "step": 6740, "unclipped_grad_norm": 0.9710240364074707 }, { "bits_per_byte": 1.511069151496585, "learning_rate": 4.762431039679047e-05, "loss_nats": 1.0473933219909668, "step": 6750, "unclipped_grad_norm": 0.9612263441085815 }, { "bits_per_byte": 1.4780942299160553, "learning_rate": 4.734962711031755e-05, "loss_nats": 1.0245368480682373, "step": 6760, "unclipped_grad_norm": 0.9238441586494446 }, { "bits_per_byte": 1.3767874194686909, "learning_rate": 4.707695414410892e-05, "loss_nats": 0.9543163180351257, "step": 6770, "unclipped_grad_norm": 0.9215719699859619 }, { "bits_per_byte": 1.4785716537548506, "learning_rate": 4.680629615739322e-05, "loss_nats": 1.0248677730560303, "step": 6780, "unclipped_grad_norm": 0.9403936266899109 }, { "bits_per_byte": 1.4459957319243224, "learning_rate": 4.653765777496863e-05, "loss_nats": 1.0022878646850586, "step": 6790, "unclipped_grad_norm": 0.9651554822921753 }, { "bits_per_byte": 1.5175859181030786, "learning_rate": 4.6271043587123986e-05, "loss_nats": 1.051910400390625, "step": 6800, "unclipped_grad_norm": 0.9926899075508118 }, { "bits_per_byte": 1.5110634760691066, "learning_rate": 4.600645814956006e-05, "loss_nats": 1.0473893880844116, "step": 6810, "unclipped_grad_norm": 0.9793192148208618 }, { "bits_per_byte": 1.5052492585913877, "learning_rate": 4.574390598331204e-05, "loss_nats": 1.0433592796325684, "step": 6820, "unclipped_grad_norm": 1.0081095695495605 }, { "bits_per_byte": 1.466343515295854, "learning_rate": 4.548339157467199e-05, "loss_nats": 1.0163918733596802, "step": 6830, "unclipped_grad_norm": 0.960361659526825 }, { "bits_per_byte": 1.4376153613130795, "learning_rate": 4.52249193751124e-05, "loss_nats": 0.9964790344238281, "step": 6840, "unclipped_grad_norm": 0.9702624678611755 }, { "bits_per_byte": 1.4922177891702109, "learning_rate": 4.4968493801209985e-05, "loss_nats": 1.0343265533447266, "step": 6850, "unclipped_grad_norm": 0.9893983006477356 }, { "bits_per_byte": 1.4659078832412185, "learning_rate": 4.471411923457034e-05, "loss_nats": 1.016089916229248, "step": 6860, "unclipped_grad_norm": 0.9693014621734619 }, { "bits_per_byte": 1.413224953750367, "learning_rate": 4.446180002175291e-05, "loss_nats": 0.9795728921890259, "step": 6870, "unclipped_grad_norm": 0.9252057075500488 }, { "bits_per_byte": 1.4200107012228098, "learning_rate": 4.421154047419693e-05, "loss_nats": 0.9842764139175415, "step": 6880, "unclipped_grad_norm": 0.8884434700012207 }, { "bits_per_byte": 1.4366857090938405, "learning_rate": 4.396334486814753e-05, "loss_nats": 0.9958346486091614, "step": 6890, "unclipped_grad_norm": 0.9679457545280457 }, { "bits_per_byte": 1.5416700245646047, "learning_rate": 4.371721744458285e-05, "loss_nats": 1.0686042308807373, "step": 6900, "unclipped_grad_norm": 0.9985603094100952 }, { "bits_per_byte": 1.4893800754309745, "learning_rate": 4.347316240914145e-05, "loss_nats": 1.0323596000671387, "step": 6910, "unclipped_grad_norm": 0.9936915636062622 }, { "bits_per_byte": 1.4499423897963222, "learning_rate": 4.3231183932050546e-05, "loss_nats": 1.00502347946167, "step": 6920, "unclipped_grad_norm": 0.923648476600647 }, { "bits_per_byte": 1.4406846496917027, "learning_rate": 4.299128614805464e-05, "loss_nats": 0.9986065030097961, "step": 6930, "unclipped_grad_norm": 0.9748247861862183 }, { "bits_per_byte": 1.408974058568991, "learning_rate": 4.2753473156344965e-05, "loss_nats": 0.9766263961791992, "step": 6940, "unclipped_grad_norm": 0.9357670545578003 }, { "bits_per_byte": 1.434796565665438, "learning_rate": 4.251774902048942e-05, "loss_nats": 0.9945251941680908, "step": 6950, "unclipped_grad_norm": 0.9624595642089844 }, { "bits_per_byte": 1.49675830313564, "learning_rate": 4.228411776836306e-05, "loss_nats": 1.0374737977981567, "step": 6960, "unclipped_grad_norm": 0.9387544989585876 }, { "bits_per_byte": 1.4929531869853003, "learning_rate": 4.2052583392079427e-05, "loss_nats": 1.0348362922668457, "step": 6970, "unclipped_grad_norm": 0.9343914985656738 }, { "bits_per_byte": 1.4660633555575984, "learning_rate": 4.182314984792214e-05, "loss_nats": 1.016197681427002, "step": 6980, "unclipped_grad_norm": 0.9326976537704468 }, { "bits_per_byte": 1.492015881538098, "learning_rate": 4.159582105627748e-05, "loss_nats": 1.034186601638794, "step": 6990, "unclipped_grad_norm": 1.0069916248321533 }, { "bits_per_byte": 1.52302985693349, "learning_rate": 4.137060090156724e-05, "loss_nats": 1.0556838512420654, "step": 7000, "unclipped_grad_norm": 0.9719612002372742 }, { "bits_per_byte": 1.512785538352196, "learning_rate": 4.114749323218249e-05, "loss_nats": 1.0485830307006836, "step": 7010, "unclipped_grad_norm": 0.9873834848403931 }, { "bits_per_byte": 1.4473323810868282, "learning_rate": 4.0926501860417706e-05, "loss_nats": 1.0032143592834473, "step": 7020, "unclipped_grad_norm": 0.939332127571106 }, { "bits_per_byte": 1.5166639191118032, "learning_rate": 4.070763056240571e-05, "loss_nats": 1.0512713193893433, "step": 7030, "unclipped_grad_norm": 0.9837853312492371 }, { "bits_per_byte": 1.5072451172546506, "learning_rate": 4.049088307805306e-05, "loss_nats": 1.0447427034378052, "step": 7040, "unclipped_grad_norm": 0.9734936952590942 }, { "bits_per_byte": 1.4796651194490362, "learning_rate": 4.027626311097629e-05, "loss_nats": 1.0256257057189941, "step": 7050, "unclipped_grad_norm": 1.005554437637329 }, { "bits_per_byte": 1.4927048440374544, "learning_rate": 4.006377432843846e-05, "loss_nats": 1.0346641540527344, "step": 7060, "unclipped_grad_norm": 0.9903765320777893 }, { "bits_per_byte": 1.4561590466770602, "learning_rate": 3.985342036128656e-05, "loss_nats": 1.009332537651062, "step": 7070, "unclipped_grad_norm": 1.009922981262207 }, { "bits_per_byte": 1.4353391709289103, "learning_rate": 3.964520480388956e-05, "loss_nats": 0.9949012994766235, "step": 7080, "unclipped_grad_norm": 0.9552373290061951 }, { "bits_per_byte": 1.4780041110070032, "learning_rate": 3.943913121407678e-05, "loss_nats": 1.0244743824005127, "step": 7090, "unclipped_grad_norm": 1.0046122074127197 }, { "bits_per_byte": 1.4728850474040718, "learning_rate": 3.923520311307735e-05, "loss_nats": 1.0209261178970337, "step": 7100, "unclipped_grad_norm": 0.9868813157081604 }, { "bits_per_byte": 1.4862149067244954, "learning_rate": 3.9033423985459816e-05, "loss_nats": 1.030165672302246, "step": 7110, "unclipped_grad_norm": 0.9593554735183716 }, { "bits_per_byte": 1.5328834309320238, "learning_rate": 3.88337972790728e-05, "loss_nats": 1.062513828277588, "step": 7120, "unclipped_grad_norm": 1.0129024982452393 }, { "bits_per_byte": 1.4811555211014134, "learning_rate": 3.8636326404985845e-05, "loss_nats": 1.0266587734222412, "step": 7130, "unclipped_grad_norm": 1.0546875 }, { "bits_per_byte": 1.4724865636020226, "learning_rate": 3.844101473743143e-05, "loss_nats": 1.0206499099731445, "step": 7140, "unclipped_grad_norm": 0.9776664972305298 }, { "bits_per_byte": 1.4357171887955067, "learning_rate": 3.824786561374701e-05, "loss_nats": 0.9951633214950562, "step": 7150, "unclipped_grad_norm": 0.9291411638259888 }, { "bits_per_byte": 1.5232358921492237, "learning_rate": 3.805688233431824e-05, "loss_nats": 1.0558266639709473, "step": 7160, "unclipped_grad_norm": 1.0189123153686523 }, { "bits_per_byte": 1.4618840051589814, "learning_rate": 3.786806816252236e-05, "loss_nats": 1.0133007764816284, "step": 7170, "unclipped_grad_norm": 0.9592419266700745 }, { "bits_per_byte": 1.4762450724539784, "learning_rate": 3.768142632467269e-05, "loss_nats": 1.0232551097869873, "step": 7180, "unclipped_grad_norm": 0.9680334329605103 }, { "bits_per_byte": 1.4441101141402626, "learning_rate": 3.7496960009963236e-05, "loss_nats": 1.0009808540344238, "step": 7190, "unclipped_grad_norm": 1.0148857831954956 }, { "bits_per_byte": 1.472608843266786, "learning_rate": 3.7314672370414325e-05, "loss_nats": 1.0207346677780151, "step": 7200, "unclipped_grad_norm": 0.9935786128044128 }, { "bits_per_byte": 1.5299951542934338, "learning_rate": 3.713456652081882e-05, "loss_nats": 1.060511827468872, "step": 7210, "unclipped_grad_norm": 0.9725626111030579 }, { "bits_per_byte": 1.5238232129019202, "learning_rate": 3.695664553868873e-05, "loss_nats": 1.0562337636947632, "step": 7220, "unclipped_grad_norm": 1.0141370296478271 }, { "bits_per_byte": 1.4390708504873315, "learning_rate": 3.678091246420277e-05, "loss_nats": 0.9974879026412964, "step": 7230, "unclipped_grad_norm": 1.0297694206237793 }, { "bits_per_byte": 1.4632025961431465, "learning_rate": 3.660737030015427e-05, "loss_nats": 1.0142147541046143, "step": 7240, "unclipped_grad_norm": 0.9345177412033081 }, { "bits_per_byte": 1.5031187375124992, "learning_rate": 3.6436022011900056e-05, "loss_nats": 1.0418825149536133, "step": 7250, "unclipped_grad_norm": 1.0408201217651367 }, { "bits_per_byte": 1.3820238612308362, "learning_rate": 3.626687052730954e-05, "loss_nats": 0.9579459428787231, "step": 7260, "unclipped_grad_norm": 0.9544695615768433 }, { "bits_per_byte": 1.4274823155068936, "learning_rate": 3.6099918736714955e-05, "loss_nats": 0.9894553422927856, "step": 7270, "unclipped_grad_norm": 0.9940797686576843 }, { "bits_per_byte": 1.4990891840027836, "learning_rate": 3.5935169492861716e-05, "loss_nats": 1.0390894412994385, "step": 7280, "unclipped_grad_norm": 1.013985514640808 }, { "bits_per_byte": 1.4370805812602216, "learning_rate": 3.5772625610859886e-05, "loss_nats": 0.99610835313797, "step": 7290, "unclipped_grad_norm": 0.9820091724395752 }, { "bits_per_byte": 1.4451605841717328, "learning_rate": 3.5612289868135887e-05, "loss_nats": 1.001708984375, "step": 7300, "unclipped_grad_norm": 0.9996907114982605 }, { "bits_per_byte": 1.4502488628801598, "learning_rate": 3.545416500438522e-05, "loss_nats": 1.0052359104156494, "step": 7310, "unclipped_grad_norm": 0.9723894000053406 }, { "bits_per_byte": 1.4141343980081293, "learning_rate": 3.52982537215255e-05, "loss_nats": 0.9802032709121704, "step": 7320, "unclipped_grad_norm": 0.9815637469291687 }, { "bits_per_byte": 1.429772436485783, "learning_rate": 3.5144558683650396e-05, "loss_nats": 0.9910427331924438, "step": 7330, "unclipped_grad_norm": 1.0089058876037598 }, { "bits_per_byte": 1.4033304478809279, "learning_rate": 3.499308251698404e-05, "loss_nats": 0.9727145433425903, "step": 7340, "unclipped_grad_norm": 0.9830491542816162 }, { "bits_per_byte": 1.4800763299672492, "learning_rate": 3.484382780983616e-05, "loss_nats": 1.02591073513031, "step": 7350, "unclipped_grad_norm": 0.9842677712440491 }, { "bits_per_byte": 1.4409101189469837, "learning_rate": 3.469679711255795e-05, "loss_nats": 0.9987627863883972, "step": 7360, "unclipped_grad_norm": 1.0266367197036743 }, { "bits_per_byte": 1.4700960047550296, "learning_rate": 3.4551992937498284e-05, "loss_nats": 1.0189929008483887, "step": 7370, "unclipped_grad_norm": 0.9935671091079712 }, { "bits_per_byte": 1.4111584102175994, "learning_rate": 3.440941775896104e-05, "loss_nats": 0.9781404733657837, "step": 7380, "unclipped_grad_norm": 0.9791932702064514 }, { "bits_per_byte": 1.4805810990475317, "learning_rate": 3.4269074013162596e-05, "loss_nats": 1.0262606143951416, "step": 7390, "unclipped_grad_norm": 0.9806758761405945 }, { "bits_per_byte": 1.422500752033327, "learning_rate": 3.41309640981904e-05, "loss_nats": 0.9860023856163025, "step": 7400, "unclipped_grad_norm": 1.014562726020813 }, { "bits_per_byte": 1.419007526420327, "learning_rate": 3.3995090373961795e-05, "loss_nats": 0.9835810661315918, "step": 7410, "unclipped_grad_norm": 0.9609370231628418 }, { "bits_per_byte": 1.483305992167802, "learning_rate": 3.3861455162183876e-05, "loss_nats": 1.0281493663787842, "step": 7420, "unclipped_grad_norm": 0.9924190640449524 }, { "bits_per_byte": 1.4892452410326982, "learning_rate": 3.3730060746313664e-05, "loss_nats": 1.0322661399841309, "step": 7430, "unclipped_grad_norm": 1.0244429111480713 }, { "bits_per_byte": 1.50888548777858, "learning_rate": 3.3600909371519225e-05, "loss_nats": 1.0458797216415405, "step": 7440, "unclipped_grad_norm": 1.0394790172576904 }, { "bits_per_byte": 1.4553271665948369, "learning_rate": 3.3474003244641194e-05, "loss_nats": 1.0087559223175049, "step": 7450, "unclipped_grad_norm": 0.99248868227005 }, { "bits_per_byte": 1.4425173828106221, "learning_rate": 3.334934453415514e-05, "loss_nats": 0.999876856803894, "step": 7460, "unclipped_grad_norm": 1.0004469156265259 }, { "bits_per_byte": 1.437790267669007, "learning_rate": 3.3226935370134486e-05, "loss_nats": 0.9966002702713013, "step": 7470, "unclipped_grad_norm": 1.0109968185424805 }, { "bits_per_byte": 1.4682612938356252, "learning_rate": 3.31067778442141e-05, "loss_nats": 1.017721176147461, "step": 7480, "unclipped_grad_norm": 1.0299010276794434 }, { "bits_per_byte": 1.5222762289574092, "learning_rate": 3.298887400955457e-05, "loss_nats": 1.055161476135254, "step": 7490, "unclipped_grad_norm": 1.0327476263046265 }, { "bits_per_byte": 1.4747042798848986, "learning_rate": 3.287322588080714e-05, "loss_nats": 1.0221871137619019, "step": 7500, "unclipped_grad_norm": 0.9886637926101685 }, { "bits_per_byte": 1.4352613487793948, "learning_rate": 3.275983543407924e-05, "loss_nats": 0.9948473572731018, "step": 7510, "unclipped_grad_norm": 0.9710773825645447 }, { "bits_per_byte": 1.4570311706995855, "learning_rate": 3.2648704606900735e-05, "loss_nats": 1.009937047958374, "step": 7520, "unclipped_grad_norm": 1.0144232511520386 }, { "bits_per_byte": 1.502727820947088, "learning_rate": 3.253983529819086e-05, "loss_nats": 1.0416115522384644, "step": 7530, "unclipped_grad_norm": 1.0307852029800415 }, { "bits_per_byte": 1.4572943041554056, "learning_rate": 3.243322936822571e-05, "loss_nats": 1.0101194381713867, "step": 7540, "unclipped_grad_norm": 0.9732069969177246 }, { "bits_per_byte": 1.4747278415080667, "learning_rate": 3.232888863860648e-05, "loss_nats": 1.0222034454345703, "step": 7550, "unclipped_grad_norm": 1.0250486135482788 }, { "bits_per_byte": 1.4513507557242378, "learning_rate": 3.222681489222838e-05, "loss_nats": 1.0059996843338013, "step": 7560, "unclipped_grad_norm": 1.0031380653381348 }, { "bits_per_byte": 1.4127096077370564, "learning_rate": 3.212700987325009e-05, "loss_nats": 0.979215681552887, "step": 7570, "unclipped_grad_norm": 0.9720339775085449 }, { "bits_per_byte": 1.587541409185384, "learning_rate": 3.202947528706403e-05, "loss_nats": 1.1003998517990112, "step": 7580, "unclipped_grad_norm": 1.080464482307434 }, { "bits_per_byte": 1.4421111597892842, "learning_rate": 3.1934212800267116e-05, "loss_nats": 0.9995952844619751, "step": 7590, "unclipped_grad_norm": 0.9815735220909119 }, { "bits_per_byte": 1.4267507013101233, "learning_rate": 3.1841224040632484e-05, "loss_nats": 0.9889482259750366, "step": 7600, "unclipped_grad_norm": 1.029758334159851 }, { "bits_per_byte": 1.4539179407536669, "learning_rate": 3.175051059708143e-05, "loss_nats": 1.0077791213989258, "step": 7610, "unclipped_grad_norm": 1.031221628189087 }, { "bits_per_byte": 1.4028547438686412, "learning_rate": 3.1662074019656414e-05, "loss_nats": 0.9723848104476929, "step": 7620, "unclipped_grad_norm": 0.9562389850616455 }, { "bits_per_byte": 1.3938100042771153, "learning_rate": 3.157591581949455e-05, "loss_nats": 0.9661154747009277, "step": 7630, "unclipped_grad_norm": 1.023301124572754 }, { "bits_per_byte": 1.4184527103886437, "learning_rate": 3.149203746880175e-05, "loss_nats": 0.983196496963501, "step": 7640, "unclipped_grad_norm": 0.9981763958930969 }, { "bits_per_byte": 1.5608753271864544, "learning_rate": 3.141044040082763e-05, "loss_nats": 1.081916332244873, "step": 7650, "unclipped_grad_norm": 1.0295648574829102 }, { "bits_per_byte": 1.4665378556913289, "learning_rate": 3.133112600984091e-05, "loss_nats": 1.0165265798568726, "step": 7660, "unclipped_grad_norm": 1.0587892532348633 }, { "bits_per_byte": 1.4931012640476933, "learning_rate": 3.125409565110571e-05, "loss_nats": 1.034938931465149, "step": 7670, "unclipped_grad_norm": 1.0342050790786743 }, { "bits_per_byte": 1.4465245785757257, "learning_rate": 3.117935064085835e-05, "loss_nats": 1.0026544332504272, "step": 7680, "unclipped_grad_norm": 1.0567405223846436 }, { "bits_per_byte": 1.4541416901824393, "learning_rate": 3.110689225628479e-05, "loss_nats": 1.0079342126846313, "step": 7690, "unclipped_grad_norm": 1.0190099477767944 }, { "bits_per_byte": 1.3949941048283057, "learning_rate": 3.1036721735498926e-05, "loss_nats": 0.9669362306594849, "step": 7700, "unclipped_grad_norm": 1.002891182899475 }, { "bits_per_byte": 1.4762163513512843, "learning_rate": 3.096884027752132e-05, "loss_nats": 1.0232352018356323, "step": 7710, "unclipped_grad_norm": 0.9984713792800903 }, { "bits_per_byte": 1.480895139368007, "learning_rate": 3.090324904225885e-05, "loss_nats": 1.0264782905578613, "step": 7720, "unclipped_grad_norm": 1.0342158079147339 }, { "bits_per_byte": 1.486845911070511, "learning_rate": 3.083994915048472e-05, "loss_nats": 1.030603051185608, "step": 7730, "unclipped_grad_norm": 1.027107834815979 }, { "bits_per_byte": 1.4387832095037634, "learning_rate": 3.077894168381942e-05, "loss_nats": 0.9972885251045227, "step": 7740, "unclipped_grad_norm": 1.0455139875411987 }, { "bits_per_byte": 1.4381068877292456, "learning_rate": 3.0720227684712254e-05, "loss_nats": 0.9968197345733643, "step": 7750, "unclipped_grad_norm": 1.0159157514572144 }, { "bits_per_byte": 1.4538223183997872, "learning_rate": 3.0663808156423456e-05, "loss_nats": 1.0077128410339355, "step": 7760, "unclipped_grad_norm": 0.9867706894874573 }, { "bits_per_byte": 1.3986834766546148, "learning_rate": 3.060968406300711e-05, "loss_nats": 0.9694935083389282, "step": 7770, "unclipped_grad_norm": 0.9943742156028748 }, { "bits_per_byte": 1.4760859885019304, "learning_rate": 3.055785632929463e-05, "loss_nats": 1.0231448411941528, "step": 7780, "unclipped_grad_norm": 1.0422483682632446 }, { "bits_per_byte": 1.4162891686407895, "learning_rate": 3.0508325840878984e-05, "loss_nats": 0.9816968441009521, "step": 7790, "unclipped_grad_norm": 0.9937946796417236 }, { "bits_per_byte": 1.45477768202533, "learning_rate": 3.046109344409957e-05, "loss_nats": 1.0083750486373901, "step": 7800, "unclipped_grad_norm": 1.0428839921951294 }, { "bits_per_byte": 1.4292601001688625, "learning_rate": 3.0416159946027727e-05, "loss_nats": 0.9906876087188721, "step": 7810, "unclipped_grad_norm": 1.028254508972168 }, { "bits_per_byte": 1.4070698666586379, "learning_rate": 3.0373526114452946e-05, "loss_nats": 0.975306510925293, "step": 7820, "unclipped_grad_norm": 1.0253064632415771 }, { "bits_per_byte": 1.4530960356651936, "learning_rate": 3.0333192677869803e-05, "loss_nats": 1.0072094202041626, "step": 7830, "unclipped_grad_norm": 1.0885143280029297 }, { "bits_per_byte": 1.4330722676078875, "learning_rate": 3.029516032546544e-05, "loss_nats": 0.9933300018310547, "step": 7840, "unclipped_grad_norm": 1.0074925422668457 }, { "bits_per_byte": 1.3858685333908742, "learning_rate": 3.02594297071078e-05, "loss_nats": 0.9606108665466309, "step": 7850, "unclipped_grad_norm": 0.9880329966545105 }, { "bits_per_byte": 1.4517502714221922, "learning_rate": 3.0226001433334603e-05, "loss_nats": 1.0062766075134277, "step": 7860, "unclipped_grad_norm": 1.0082415342330933 }, { "bits_per_byte": 1.4675547891058807, "learning_rate": 3.019487607534278e-05, "loss_nats": 1.0172314643859863, "step": 7870, "unclipped_grad_norm": 1.023923397064209 }, { "bits_per_byte": 1.3867994754799946, "learning_rate": 3.016605416497886e-05, "loss_nats": 0.9612561464309692, "step": 7880, "unclipped_grad_norm": 1.0151392221450806 }, { "bits_per_byte": 1.4568015738606837, "learning_rate": 3.0139536194729734e-05, "loss_nats": 1.0097779035568237, "step": 7890, "unclipped_grad_norm": 1.0505098104476929 }, { "bits_per_byte": 1.446776189193938, "learning_rate": 3.0115322617714362e-05, "loss_nats": 1.00282883644104, "step": 7900, "unclipped_grad_norm": 1.0279992818832397 }, { "bits_per_byte": 1.461830518554563, "learning_rate": 3.0093413847675965e-05, "loss_nats": 1.0132637023925781, "step": 7910, "unclipped_grad_norm": 1.0125948190689087 }, { "bits_per_byte": 1.4593548282953928, "learning_rate": 3.0073810258974985e-05, "loss_nats": 1.0115476846694946, "step": 7920, "unclipped_grad_norm": 1.0411789417266846 }, { "bits_per_byte": 1.4317973602170415, "learning_rate": 3.0056512186582658e-05, "loss_nats": 0.9924463033676147, "step": 7930, "unclipped_grad_norm": 1.0330348014831543 }, { "bits_per_byte": 1.431196452834927, "learning_rate": 3.0041519926075316e-05, "loss_nats": 0.9920297861099243, "step": 7940, "unclipped_grad_norm": 1.0142537355422974 }, { "bits_per_byte": 1.49434246283897, "learning_rate": 3.0028833733629313e-05, "loss_nats": 1.035799264907837, "step": 7950, "unclipped_grad_norm": 1.0716009140014648 }, { "bits_per_byte": 1.4186909923514142, "learning_rate": 3.0018453826016686e-05, "loss_nats": 0.9833616614341736, "step": 7960, "unclipped_grad_norm": 1.0115852355957031 }, { "bits_per_byte": 1.4420433126335188, "learning_rate": 3.001038038060143e-05, "loss_nats": 0.9995482563972473, "step": 7970, "unclipped_grad_norm": 1.0335451364517212 }, { "bits_per_byte": 1.4872165336831205, "learning_rate": 3.0004613535336433e-05, "loss_nats": 1.0308599472045898, "step": 7980, "unclipped_grad_norm": 1.0748283863067627 }, { "bits_per_byte": 1.5234063269562288, "learning_rate": 3.000115338876117e-05, "loss_nats": 1.055944800376892, "step": 7990, "unclipped_grad_norm": 1.076164722442627 }, { "bits_per_byte": 1.4309937712808851, "learning_rate": 3e-05, "loss_nats": 0.9918892979621887, "step": 8000, "unclipped_grad_norm": 1.0322442054748535 } ] }