{ "architecture": "block", "artifacts": { "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" }, "batch_size": 32, "canonical_sha256_without_self": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766", "depth": 32, "diagnostics": [ { "forward_identity_gate": { "comparisons": { "detached_learned": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_value_backward": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "initialization_negative_control": { "comparisons": { "detached_learned": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "uniform_value_backward": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } } }, "passed": true }, "loss_scale_gate": { "checks": { "attention_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "mlp_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_attention_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 5.551115123125783e-17, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_mlp_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_attention_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_mlp_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } } }, "passed": true }, "modes": { "detached_learned": { "bits_per_byte": 8.06433666787865, "forward": { "activation_sha256": { "attention_branch_output": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", "mlp_branch_output": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", "post_attention_state": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", "post_mlp_state": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", "pre_attention_input": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", "pre_mlp_input": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248" }, "logits_sha256": "30ae47372d19e5c17da2b2a4846b2d27ef6e44fdc06045752d040a56bc6e9398", "loss_nats": 5.5897722244262695, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.5897722244262695, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0021227982360869646, 0.00224129855632782, 0.0024903982412070036, 0.0023722455371171236, 0.0023577234242111444, 0.0023993616923689842, 0.002688874490559101, 0.0029664887115359306, 0.0029547461308538914, 0.003052416956052184, 0.0028214550111442804, 0.0032544767018407583, 0.00348988501355052, 0.003914494998753071, 0.0035071265883743763, 0.004211232997477055, 0.004072706215083599, 0.004050548188388348, 0.003929397091269493, 0.004410331137478352, 0.004824656527489424, 0.00433384021744132, 0.004475373309105635, 0.004755842965096235, 0.004843301139771938, 0.0050001065246760845, 0.005346097983419895, 0.004958339501172304, 0.0054801348596811295, 0.005455824080854654, 0.005286100786179304, 0.005402065813541412 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015727665777376387, "first_to_last_ratio": 3.444367030996094, "imbalance_abs_log_ratio": 1.2367401520242614, "last_quartile_mean": 4.566199140754179e-05, "mean": 8.846047569477378e-05, "normalized": [ 2.4938583213844665, 2.2321521039252294, 1.9412946714691228, 1.7875630514690435, 1.605255150569201, 1.4809330135294976, 1.3782219007164203, 1.3041730210393685, 1.1983684502302745, 1.1165155187552953, 1.0705163494358503, 1.028915549335682, 0.969265519878856, 0.9296915475356099, 0.8760186941641476, 0.840544602365157, 0.8215144482528359, 0.7958412144142111, 0.7578169321029499, 0.7149281644739428, 0.6789722877716906, 0.648159768212111, 0.6127699836305072, 0.5872278577962867, 0.5712871725092085, 0.5525140937217092, 0.5351005407841478, 0.5196042162209058, 0.49974197043571306, 0.49405312503381205, 0.48454882285128825, 0.4726319359854579 ], "peak_layer": 1, "peak_normalized": 2.4938583213844665, "population_cv": 0.5254866190950809, "rest_mean": 9.469065830840177e-05, "spike_contrast": 0.5789112734082525, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.48174895811826e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00022060789342503995, 0.0001974572369363159, 0.00017172785010188818, 0.00015812867786735296, 0.00014200163423083723, 0.0001310040388489142, 0.00012191816495032981, 0.00011536776582943276, 0.00010600824316497892, 9.876749390969053e-05, 9.469838551012799e-05, 9.101835894398391e-05, 8.574168896302581e-05, 8.224095654441044e-05, 7.749303040327504e-05, 7.435497536789626e-05, 7.267155888257548e-05, 7.040049240458757e-05, 6.703684630338103e-05, 6.324288551695645e-05, 6.0062211559852585e-05, 5.733652142225765e-05, 5.42059242434334e-05, 5.1946455641882494e-05, 5.053633503848687e-05, 4.887565955868922e-05, 4.733524838229641e-05, 4.596443613991141e-05, 4.420741242938675e-05, 4.370417445898056e-05, 4.286341936676763e-05, 4.180924588581547e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00022213962802197784, "first_to_last_ratio": 1.7678973829107683, "imbalance_abs_log_ratio": 0.5697909211909248, "last_quartile_mean": 0.00012565187898871955, "mean": 0.00016188367408176418, "normalized": [ 1.7325269150218519, 1.5951124185334453, 1.452057969341735, 1.3730142236731557, 1.2857984704138503, 1.2257558334519054, 1.176224442377172, 1.1372505911497117, 1.088955563427776, 1.050933480062714, 1.0259126283978994, 1.0049209692997896, 0.9780904283068869, 0.9585002626676595, 0.9364797090707767, 0.9194681619525875, 0.9067005574410794, 0.8936181540011504, 0.876961137714969, 0.8605080845266135, 0.8461083257765747, 0.8336465310750286, 0.8211746685608088, 0.8107906186997867, 0.8025238658005067, 0.7926026656489187, 0.7851375623337778, 0.7775608141673811, 0.7689788128837574, 0.7655650153667295, 0.7607140375905382, 0.7564070812634623 ], "peak_layer": 1, "peak_normalized": 1.7325269150218519, "population_cv": 0.2510633514944888, "rest_mean": 0.00016719439386010722, "spike_contrast": 0.7967120440064878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001332057872787118, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002804678224492818, 0.0002582226588856429, 0.00023506447905674577, 0.00022226858709473163, 0.00020814978051930666, 0.00019842985784634948, 0.00019041153427679092, 0.00018410230404697359, 0.00017628412751946598, 0.0001701289729680866, 0.0001660785055719316, 0.0001626802986720577, 0.0001583368721185252, 0.00015516554412897676, 0.00015160077600739896, 0.00014884688425809145, 0.0001467800175305456, 0.00014466218999587, 0.00014196569100022316, 0.0001393022103002295, 0.00013697112444788218, 0.00013495376333594322, 0.0001329347724094987, 0.0001312537642661482, 0.00012991551193408668, 0.00012830943160224706, 0.0001271009532501921, 0.00012587440141942352, 0.0001244851155206561, 0.00012393247743602842, 0.0001231471833307296, 0.00012244995741639286 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19699564762413502, "first_to_last_ratio": 1.7678974461601602, "imbalance_abs_log_ratio": 0.5697909569675436, "last_quartile_mean": 0.11142934113740921, "mean": 0.14356006286107004, "normalized": [ 1.73252692376823, 1.5951124529267644, 1.4520579427898739, 1.3730142509369438, 1.2857984860257277, 1.2257558423238033, 1.1762244498747965, 1.1372506023188744, 1.0889555406950477, 1.0509335152923371, 1.0259126331612176, 1.004920958483027, 0.9780904724894115, 0.9585002702989701, 0.9364797557971557, 0.9194681924019411, 0.9067005932791752, 0.893618176643793, 0.876961081367795, 0.8605080517791706, 0.8461083423470157, 0.8336465303631113, 0.8211746500314575, 0.8107906024915998, 0.8025238662049899, 0.792602599370704, 0.7851375422989607, 0.7775607992253795, 0.7689787783378558, 0.7655650375651303, 0.7607140142018359, 0.7564070449079043 ], "peak_layer": 1, "peak_normalized": 1.73252692376823, "population_cv": 0.25106336091926634, "rest_mean": 0.1482696624265777, "spike_contrast": 0.7967120399011176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1181282252073288, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2487216740846634, 0.2289944440126419, 0.20845752954483032, 0.19711001217365265, 0.18458931148052216, 0.17596958577632904, 0.1688588559627533, 0.16326376795768738, 0.15633052587509155, 0.15087208151817322, 0.14728008210659027, 0.1442665159702301, 0.14041472971439362, 0.13760235905647278, 0.1344410926103592, 0.1319989114999771, 0.13016599416732788, 0.1282878816127777, 0.12589658796787262, 0.12353459000587463, 0.12146736681461334, 0.11967834830284119, 0.11788788437843323, 0.11639714986085892, 0.11521037667989731, 0.11378607898950577, 0.11271439492702484, 0.11162667721509933, 0.11039464175701141, 0.10990456491708755, 0.10920815169811249, 0.10858984291553497 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.813340178268845e-05, "first_to_last_ratio": 2.0396781258635253, "imbalance_abs_log_ratio": 0.7127920139678758, "last_quartile_mean": 3.3403996894776355e-05, "mean": 4.817690887648496e-05, "normalized": [ 1.6677253734616262, 1.5880838657379246, 1.4689679818917765, 1.414042908307037, 1.3674754530876347, 1.307279124984285, 1.2642920390991146, 1.2360018052494937, 1.16456296339031, 1.101785452839103, 1.0797656603944665, 1.0672809591929815, 1.0128091896347613, 0.9955804650846075, 0.9623876307655963, 0.9380984486976066, 0.9310658554452689, 0.9082443423860135, 0.8811967490595315, 0.8644115118569954, 0.8497936463301483, 0.8309315019236491, 0.7889969150425813, 0.7623309639866356, 0.7584133539756696, 0.7445182980676538, 0.7180454335344151, 0.7000361319857387, 0.6663708133598921, 0.6665317313809671, 0.6558653811637546, 0.6371080486827607 ], "peak_layer": 1, "peak_normalized": 1.6677253734616262, "population_cv": 0.28258812904813146, "rest_mean": 4.997824995640527e-05, "spike_contrast": 0.7693279992487513, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.844966704491526e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.034585334826261e-05, 7.650897168787196e-05, 7.077033660607412e-05, 6.81242163409479e-05, 6.588074029423296e-05, 6.298066728049889e-05, 6.090968236094341e-05, 5.954674634267576e-05, 5.610504376818426e-05, 5.308061736286618e-05, 5.201977182878181e-05, 5.1418297516647726e-05, 4.879401603830047e-05, 4.796398934558965e-05, 4.6364861191250384e-05, 4.519468348007649e-05, 4.4855874875793234e-05, 4.3756404920713976e-05, 4.245333548169583e-05, 4.1644674638519064e-05, 4.094043106306344e-05, 4.0031711250776425e-05, 3.8011432479834184e-05, 3.672674938570708e-05, 3.653801104519516e-05, 3.586859020288102e-05, 3.459320942056365e-05, 3.372557694092393e-05, 3.210368595318869e-05, 3.21114384860266e-05, 3.159756670356728e-05, 3.069389640586451e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018498800636734813, "first_to_last_ratio": 1.4771173407265819, "imbalance_abs_log_ratio": 0.39009244570614415, "last_quartile_mean": 0.0001252358233614359, "mean": 0.00015057798327688943, "normalized": [ 1.36489476841379, 1.32224224178262, 1.270236278236952, 1.2358216684832493, 1.2002806183282966, 1.1699184301039023, 1.1431585124660382, 1.1216044194279113, 1.0931779382416982, 1.0676137521656621, 1.050334546837667, 1.0352083927879365, 1.0149316924423049, 1.001461761419052, 0.9840691969120363, 0.9695910166861202, 0.9584136856662008, 0.9467149797350989, 0.9322036518549593, 0.9180752132052378, 0.9048391519289843, 0.8925982934025656, 0.8798985862981495, 0.8691051109125614, 0.860352873872478, 0.849675270316085, 0.8415818308236442, 0.8332603200076834, 0.823928337260059, 0.8201054366601405, 0.8147645087933348, 0.8099375145275801 ], "peak_layer": 1, "peak_normalized": 1.36489476841379, "population_cv": 0.15663918075507588, "rest_mean": 0.0001538862707093358, "spike_contrast": 0.8624111204329009, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000132713231141679, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00020552310161292553, 0.00019910057017114013, 0.00019126961706206203, 0.0001860875345300883, 0.00018073583487421274, 0.00017616395780351013, 0.0001721345033729449, 0.00016888893151190132, 0.0001646085293032229, 0.00016075912571977824, 0.00015815725782886147, 0.0001558795920573175, 0.00015282636741176248, 0.00015079809236340225, 0.0001481791550759226, 0.00014599905989598483, 0.00014431599993258715, 0.00014255443238653243, 0.00014036934589967132, 0.000138241914100945, 0.00013624885468743742, 0.00013440565089695156, 0.0001324933546129614, 0.0001308680948568508, 0.00012955020065419376, 0.00012794238864444196, 0.0001267236948478967, 0.00012547065853141248, 0.00012406546738930047, 0.00012348982272669673, 0.0001226855965796858, 0.00012195875751785934 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015959660413500387, "first_to_last_ratio": 1.2728751185715401, "imbalance_abs_log_ratio": 0.2412782146622287, "last_quartile_mean": 0.00012538276678242255, "mean": 0.00014144585429676226, "normalized": [ 1.173955566226187, 1.1612997099437044, 1.147257651804958, 1.1326524298721496, 1.1221543736292459, 1.1091111750587395, 1.0952125313023318, 1.0849401932479867, 1.070907908686229, 1.0558579375605581, 1.0430997159190893, 1.0346471150202732, 1.0228091528121042, 1.0150746526225596, 1.0038187840838828, 0.9945868690199192, 0.986355871119351, 0.9777591356798918, 0.9676015085554662, 0.9595938626090281, 0.9511867345631487, 0.9428977124791953, 0.9335239262292595, 0.9222035520824579, 0.9142471402553919, 0.9038764491526137, 0.8952592406413572, 0.8872171299760644, 0.877779969795339, 0.8748896658804006, 0.8709399062198925, 0.8672824279512239 ], "peak_layer": 1, "peak_normalized": 1.173955566226187, "population_cv": 0.09275081554992197, "rest_mean": 0.00014320575994335942, "spike_contrast": 0.9213481626529775, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013194236380513756, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016605114797130227, 0.0001642610295675695, 0.0001622748386580497, 0.00016020899056456983, 0.00015872408403083682, 0.00015687917766626924, 0.0001549132721265778, 0.00015346029249485582, 0.00015147548401728272, 0.00014934672799427062, 0.00014754213043488562, 0.000146346545079723, 0.00014467211440205574, 0.00014357810141518712, 0.00014198600547388196, 0.00014068018936086446, 0.00013951594883110374, 0.00013829997624270618, 0.00013686322199646384, 0.0001357305736746639, 0.00013454142026603222, 0.0001333689724560827, 0.00013204308925196528, 0.00013044186925981194, 0.00012931646779179573, 0.00012784957652911544, 0.00012663070810958743, 0.000125493184896186, 0.0001241583377122879, 0.00012374951620586216, 0.0001231908390764147, 0.00012267350393813103 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031297675377572887, "first_to_last_ratio": 2.208693462790297, "imbalance_abs_log_ratio": 0.7924011473984336, "last_quartile_mean": 0.00014170221402309835, "mean": 0.00020716176595669822, "normalized": [ 1.8096319791310764, 1.7504284704923412, 1.6109023900661548, 1.5434988529701714, 1.4353862656069223, 1.3677596328902204, 1.3131374518280827, 1.2555297512219317, 1.1870915168673883, 1.122656500215855, 1.088514230207685, 1.04408820740568, 0.9901334868561585, 0.9607872809839492, 0.9219812869372472, 0.880015778430751, 0.8614610505244651, 0.8397919049431777, 0.8116424497290868, 0.7845312039842096, 0.7619415782348183, 0.7436794877201931, 0.7280047725498856, 0.715266757195506, 0.7065993237444729, 0.696468071137219, 0.6907565141722826, 0.6863265628748565, 0.6782234011874358, 0.6754056248242399, 0.6708832321424102, 0.6674749829241257 ], "peak_layer": 1, "peak_normalized": 1.8096319791310764, "population_cv": 0.33826602846938675, "rest_mean": 0.00021747771663487784, "spike_contrast": 0.6964190843920172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001514556322945282, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00037488655652850866, 0.0003626218531280756, 0.0003337173839099705, 0.0003197539481334388, 0.0002973571536131203, 0.00028334750095382333, 0.00027203187346458435, 0.0002600977604743093, 0.0002459199749864638, 0.00023257150314748287, 0.00022549853019881994, 0.0002162951568607241, 0.00020511780166998506, 0.00019903838983736932, 0.00019099927158094943, 0.00018230562272947282, 0.00017846179252956063, 0.00017397277406416833, 0.0001681412832112983, 0.0001625248696655035, 0.0001578451629029587, 0.00015406195598188788, 0.00015081475430633873, 0.0001481759245507419, 0.00014638036373071373, 0.0001442815555492416, 0.0001430983393220231, 0.00014218062278814614, 0.00014050195750314742, 0.00013991822197567672, 0.00013898135512135923, 0.00013827529619447887 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027448106557130814, 0.0027243648655712605, 0.0027103275060653687, 0.0027234479784965515, 0.0027595730498433113, 0.0027272943407297134, 0.002728298306465149, 0.0027007071767002344, 0.002684502862393856, 0.002683063969016075, 0.0027139110025018454, 0.002666700165718794, 0.002679243916645646, 0.0026584581937640905, 0.002684473292902112, 0.0026407993864268064, 0.002663099905475974, 0.0026283918414264917, 0.002597011858597398, 0.0026434636674821377, 0.0025708195753395557, 0.002691218862310052, 0.0026598076801747084, 0.0026596153620630503, 0.0026765079237520695, 0.002582729561254382, 0.002614464145153761, 0.0025760054122656584, 0.002634295728057623, 0.00259007909335196, 0.00257018837146461, 0.0025643527042120695 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.000156725715896755, "first_to_last_ratio": 3.4451295459637388, "imbalance_abs_log_ratio": 1.236961507811942, "last_quartile_mean": 4.549196592051885e-05, "mean": 8.812961027615529e-05, "normalized": [ 2.4952457548653, 2.231881013356295, 1.939989240144028, 1.7866526882708733, 1.6079164676041713, 1.4797016884434102, 1.3811975174028512, 1.3042540698565042, 1.1954344722384815, 1.1168849900912188, 1.0692220969851183, 1.0304138184362992, 0.9676518223611118, 0.9260610329319834, 0.8738547948342114, 0.8395441231817105, 0.8221010666653766, 0.7971164256793037, 0.7591159941756684, 0.7162115141810864, 0.6791638220644127, 0.6489084022271779, 0.6150954832405342, 0.5868306199881643, 0.5699702288291514, 0.552179306852942, 0.5335984532948914, 0.5198944860736878, 0.5024723171712969, 0.4942325662695092, 0.4850276094601851, 0.47217611282304317 ], "peak_layer": 1, "peak_normalized": 2.4952457548653, "population_cv": 0.5254777238950282, "rest_mean": 9.433142622583546e-05, "spike_contrast": 0.5792322488273548, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.463980414788239e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021990503591950983, 0.00019669480388984084, 0.00017097049567382783, 0.00015745700511615723, 0.00014170505164656788, 0.0001304055331274867, 0.00012172439892310649, 0.00011494340287754312, 0.00010535317414905876, 9.843063890002668e-05, 9.423012670595199e-05, 9.080996824195608e-05, 8.527877798769623e-05, 8.16133979242295e-05, 7.701248250668868e-05, 7.398869638564065e-05, 7.24514466128312e-05, 7.024955993983895e-05, 6.690059672109783e-05, 6.311944162007421e-05, 5.985444295220077e-05, 5.718804459320381e-05, 5.420812522061169e-05, 5.1717153837671503e-05, 5.023125413572416e-05, 4.866334711550735e-05, 4.7025823732838035e-05, 4.581809844239615e-05, 4.428268948686309e-05, 4.3556523451115936e-05, 4.274529419490136e-05, 4.161269680480473e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002203928706876468, "first_to_last_ratio": 1.7624395804596433, "imbalance_abs_log_ratio": 0.5666989745264205, "last_quartile_mean": 0.00012504988717410015, "mean": 0.00016078790008577926, "normalized": [ 1.7301295632680096, 1.5926765618359837, 1.4498368649995454, 1.3715838442653223, 1.2853282969253275, 1.2242662006350955, 1.1759759999659367, 1.1358472505674493, 1.0864560712260232, 1.0500344477950443, 1.0256300088344572, 1.0049332388641516, 0.9775387355078093, 0.9578140668049778, 0.9358065303644342, 0.9184543281568608, 0.9072826301440372, 0.8934865935730769, 0.8779252804708734, 0.8613603707803614, 0.8467225584074481, 0.8352602527072714, 0.8224752340358507, 0.8113194646910352, 0.80159772798593, 0.7940537841119606, 0.785769247679408, 0.7781743501685717, 0.7724121545166266, 0.7676777199663887, 0.7637141967462265, 0.7584564239985051 ], "peak_layer": 1, "peak_normalized": 1.7301295632680096, "population_cv": 0.2498913679417364, "rest_mean": 0.0001660440253477578, "spike_contrast": 0.7974079368034485, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013240482367109506, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027818389935418963, 0.00025608311989344656, 0.00023311622499022633, 0.00022053408611100167, 0.00020666523778345436, 0.0001968471915461123, 0.00018908271158579737, 0.0001826304942369461, 0.0001746889902278781, 0.00016883283387869596, 0.00016490889538545161, 0.00016158110520336777, 0.00015717640053480864, 0.00015400491247419268, 0.0001504663669038564, 0.00014767634274903685, 0.00014588006888516247, 0.00014366183313541114, 0.00014115976227913052, 0.00013849632523488253, 0.00013614274212159216, 0.0001342997420579195, 0.00013224406575318426, 0.00013045035302639008, 0.00012888721539638937, 0.00012767424050252885, 0.0001263421872863546, 0.0001251210196642205, 0.00012419452832546085, 0.00012343328853603452, 0.00012279600196052343, 0.00012195061572128907 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19544659927487373, "first_to_last_ratio": 1.7624395617194093, "imbalance_abs_log_ratio": 0.5666989638932991, "last_quartile_mean": 0.11089549027383327, "mean": 0.14258831785991788, "normalized": [ 1.7301295593799786, 1.5926765086027146, 1.4498368632986096, 1.3715838909156086, 1.2853282557768668, 1.224266111161914, 1.1759760185820738, 1.1358472270183295, 1.0864560720464596, 1.050034483149275, 1.025630003735948, 1.0049332495130952, 0.9775386792530747, 0.9578141330250067, 0.9358065753109587, 0.9184542876534186, 0.9072827263675407, 0.8934866319653527, 0.8779253472761699, 0.8613603464001749, 0.8467225704769723, 0.8352602246328348, 0.8224752140046494, 0.8113194329409806, 0.8015977187598701, 0.7940537797484172, 0.7857692153409586, 0.7781743825002613, 0.7724121455314925, 0.7676776608495898, 0.7637142124554999, 0.758456472325904 ], "peak_layer": 1, "peak_normalized": 1.7301295593799786, "population_cv": 0.24989136150327743, "rest_mean": 0.14724950271624107, "spike_contrast": 0.7974079196861148, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11741791963577271, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24669626355171204, 0.2270970642566681, 0.20672979950904846, 0.19557183980941772, 0.18327279388904572, 0.17456604540348053, 0.16768044233322144, 0.16195854544639587, 0.1549159437417984, 0.1497226506471634, 0.14624285697937012, 0.14329174160957336, 0.13938559591770172, 0.13657310605049133, 0.13343508541584015, 0.1309608519077301, 0.1293679177761078, 0.12740075588226318, 0.1251818984746933, 0.1228199228644371, 0.12073274701833725, 0.11909835040569305, 0.11727535724639893, 0.11568467319011688, 0.11429847031831741, 0.11322279274463654, 0.11204151064157486, 0.11095857620239258, 0.11013694852590561, 0.10946186631917953, 0.10889672487974167, 0.108147032558918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.812671563238837e-05, "first_to_last_ratio": 2.0429267595554257, "imbalance_abs_log_ratio": 0.7143834657339574, "last_quartile_mean": 3.334760549478233e-05, "mean": 4.8136708642232406e-05, "normalized": [ 1.6706878479333491, 1.5890573277727331, 1.4709453763829752, 1.4135957551070697, 1.368779351628352, 1.3072249852707734, 1.2659085053079855, 1.2360067186760548, 1.1647282180273046, 1.1006681301372137, 1.0750864931045292, 1.0685694254166505, 1.0112110373273602, 0.9945647481091522, 0.9602230239537092, 0.9352227929155844, 0.93139607887314, 0.9092220870879134, 0.8814974932144994, 0.8653605860843365, 0.8506124631158822, 0.8320765477333729, 0.7932580049774197, 0.7619473309737773, 0.7566723544753674, 0.7419402536141336, 0.7141691805145849, 0.7010673290630277, 0.6716446941281601, 0.6659218543873533, 0.6557479674543484, 0.6349860372318865 ], "peak_layer": 1, "peak_normalized": 1.6706878479333491, "population_cv": 0.2830124551063321, "rest_mean": 4.992923641111702e-05, "spike_contrast": 0.7702312603701833, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.845705869025551e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.04214141680859e-05, 7.649198960280046e-05, 7.080646901158616e-05, 6.804584700148553e-05, 6.588853284483775e-05, 6.292550824582577e-05, 6.093666888773441e-05, 5.949729529675096e-05, 5.6066182878566906e-05, 5.298254109220579e-05, 5.1751125283772126e-05, 5.143741509527899e-05, 4.867637107963674e-05, 4.7875073505565524e-05, 4.622197593562305e-05, 4.501854709815234e-05, 4.483434167923406e-05, 4.376695869723335e-05, 4.24323879997246e-05, 4.165561040281318e-05, 4.094568430446088e-05, 4.0053426346275955e-05, 3.8184829463716596e-05, 3.667763667181134e-05, 3.642371666501276e-05, 3.571456181816757e-05, 3.4377753763692454e-05, 3.3747073757695034e-05, 3.2330764952348545e-05, 3.205528628313914e-05, 3.156554885208607e-05, 3.056613786611706e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018337754227104597, "first_to_last_ratio": 1.4712162101549686, "imbalance_abs_log_ratio": 0.38608941256902984, "last_quartile_mean": 0.00012464350311347516, "mean": 0.00014952537844692415, "normalized": [ 1.3620834570997993, 1.3194965005671844, 1.2677944868152182, 1.233561928872588, 1.1984447257225705, 1.1684695593357564, 1.141528073961558, 1.1198008373135866, 1.0910012077210556, 1.066096352737946, 1.0498630645220353, 1.0350823851472097, 1.0145919973375974, 1.0013407128400098, 0.9836056688270479, 0.9685428567287663, 0.9591027483551056, 0.9465833155582388, 0.9334001555540693, 0.9191584382428863, 0.9057431683575706, 0.8945627711391904, 0.8815040843165663, 0.8698871064160805, 0.8596264871795505, 0.8514885298372257, 0.8425068993042694, 0.8341757603627517, 0.8278236379550317, 0.8225948884437775, 0.8181940461689637, 0.812344147258793 ], "peak_layer": 1, "peak_normalized": 1.3620834570997993, "population_cv": 0.15505530299928885, "rest_mean": 0.00015278545470209792, "spike_contrast": 0.8634393039979241, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001319209666689858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00020366604439914227, 0.0001972982136067003, 0.0001895674504339695, 0.00018444881425239146, 0.00017919790116138756, 0.00017471585306338966, 0.0001706874172668904, 0.0001674386439844966, 0.00016313236847054213, 0.00015940846060402691, 0.00015698117204010487, 0.00015477108536288142, 0.00015170725237112492, 0.00014972584904171526, 0.0001470740098739043, 0.0001448217371944338, 0.00014341020141728222, 0.00014153822849038988, 0.00013956701150164008, 0.00013743751333095133, 0.00013543159002438188, 0.00013375983689911664, 0.00013180723180994391, 0.00013007019879296422, 0.00012853597581852227, 0.00012731914466712624, 0.0001259761629626155, 0.00012473044625949115, 0.00012378064275253564, 0.00012299881200306118, 0.00012234077439643443, 0.00012146606604801491 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015816818631719798, "first_to_last_ratio": 1.2671705792413261, "imbalance_abs_log_ratio": 0.23678652466939742, "last_quartile_mean": 0.00012481996418500785, "mean": 0.0001404537983944465, "normalized": [ 1.172032188999453, 1.1600674087975105, 1.1434054243316032, 1.1305758413418425, 1.1202630655069494, 1.1065457827105485, 1.0931998253234612, 1.0828906757132684, 1.0683311763286112, 1.0526488941703125, 1.0427563459749776, 1.0343706495522589, 1.0219941374371992, 1.0154952175292713, 1.0038357654470964, 0.9937321709742023, 0.9862865992066487, 0.9768881500283385, 0.968261257859838, 0.9607910277643221, 0.9520546236098718, 0.9457805296333641, 0.9349892993508679, 0.9232794945485685, 0.9140411172891727, 0.9051057888083044, 0.8959794100889387, 0.888100763485746, 0.8822517666505367, 0.8784085901067091, 0.87543446421392, 0.8702025472162866 ], "peak_layer": 1, "peak_normalized": 1.172032188999453, "population_cv": 0.09096684083762872, "rest_mean": 0.00014216970130628734, "spike_contrast": 0.9227558436510851, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013118792267050595, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016461637278553098, 0.0001629358739592135, 0.00016059563495218754, 0.00015879367128945887, 0.0001573452027514577, 0.00015541855827905238, 0.00015354406787082553, 0.00015209610864985734, 0.00015005117165856063, 0.0001478485355619341, 0.0001464590895920992, 0.00014528128667734563, 0.00014354295853991061, 0.00014263016055338085, 0.00014099254622124135, 0.00013957345800008625, 0.0001385276991641149, 0.00013720765127800405, 0.00013599597150459886, 0.00013494674931280315, 0.00013371968816500157, 0.00013283846783451736, 0.00013132279855199158, 0.0001296781119890511, 0.0001283805468119681, 0.00012712554598692805, 0.0001258437114302069, 0.00012473712558858097, 0.00012391561176627874, 0.0001233758230227977, 0.0001229580957442522, 0.00012222325312905014 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0003110146244580392, "first_to_last_ratio": 2.2098355110548322, "imbalance_abs_log_ratio": 0.7929180833727034, "last_quartile_mean": 0.00014074107457417995, "mean": 0.00020577548139044666, "normalized": [ 1.8085529821987474, 1.7406694730308092, 1.619926374791616, 1.5420513399817848, 1.4477144166533082, 1.3679714513872108, 1.309931951281633, 1.254598241859695, 1.1926238819658395, 1.119002359728262, 1.082854716042971, 1.0468411526236405, 0.9850613364068194, 0.9618428208148979, 0.9182052809224994, 0.877960480924469, 0.8608981306240351, 0.8408442911829569, 0.8148282643879068, 0.7879356949258838, 0.7604349554811285, 0.7438321978222967, 0.727404890131739, 0.7163770004088065, 0.7034258791916428, 0.6971552930824737, 0.6907054382632783, 0.684504508832364, 0.6799757639842041, 0.6759922505973077, 0.6720742215597147, 0.6678029589100581 ], "peak_layer": 1, "peak_normalized": 1.8085529821987474, "population_cv": 0.3383164219384334, "rest_mean": 0.00021605301460820355, "spike_contrast": 0.695555219569954, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001502768020145595, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00037215586053207517, 0.0003581870987545699, 0.00033334112958982587, 0.0003173163568135351, 0.00029790413100272417, 0.0002814949839375913, 0.00026955187786370516, 0.0002581655571702868, 0.00024541275342926383, 0.00023026324925012887, 0.00022282495046965778, 0.00021541424212045968, 0.00020270147069822997, 0.00019792366947513074, 0.00018894413369707763, 0.00018066274060402066, 0.00017715172725729644, 0.00017302513879258186, 0.00016767167835496366, 0.00016213784692808986, 0.0001564788690302521, 0.00015306242858059704, 0.00014968209143262357, 0.00014741282211616635, 0.00014474779891315848, 0.00014345746603794396, 0.00014213024405762553, 0.00014085424481891096, 0.00013992234016768634, 0.00013910263078287244, 0.00013829639647156, 0.00013741747534368187 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0021227982360869646, 0.00413061399012804, 0.005416445434093475, 0.006388718727976084, 0.0023577234242111444, 0.004388594534248114, 0.00574506726115942, 0.007066869642585516, 0.0029547461308538914, 0.004991721361875534, 0.006190058775246143, 0.0074937604367733, 0.00348988501355052, 0.006057556718587875, 0.0074848332442343235, 0.00897275935858488, 0.004072706215083599, 0.0063449000008404255, 0.007735534105449915, 0.00946879480034113, 0.004824656527489424, 0.007074959110468626, 0.009095527231693268, 0.010346688330173492, 0.004843301139771938, 0.007334613241255283, 0.009146629832684994, 0.010587968863546848, 0.0054801348596811295, 0.0077637480571866035, 0.009536821395158768, 0.011367839761078358 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015727686513855588, "first_to_last_ratio": 3.444355141493678, "imbalance_abs_log_ratio": 1.2367367001496994, "last_quartile_mean": 4.5662209231522866e-05, "mean": 8.846079299473786e-05, "normalized": [ 2.4938513501811554, 2.232208910931446, 1.9412909982664082, 1.7875424925486438, 1.6051944492471113, 1.4809426711898561, 1.3782256757419, 1.304162421055249, 1.1983748443875557, 1.116540877408076, 1.0705280549740306, 1.028906183424365, 0.9692497056236429, 0.9297086109864628, 0.876031508599092, 0.840542245421429, 0.8215073067800788, 0.7958492991481336, 0.7578128978794811, 0.7149268338582802, 0.6789684541053794, 0.6481555926886391, 0.6127683614407704, 0.5872234895762031, 0.5712935129257742, 0.5525148673059793, 0.5351006365732414, 0.5196040797164857, 0.49974182292452146, 0.4940518464188538, 0.4845470025727643, 0.4726329960989905 ], "peak_layer": 1, "peak_normalized": 2.4938513501811554, "population_cv": 0.525487396006855, "rest_mean": 9.469102304946218e-05, "spike_contrast": 0.5789096889426617, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.481755069922656e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002206080680480227, 0.00019746297039091587, 0.00017172814114019275, 0.00015812742640264332, 0.00014199677389115095, 0.0001310053630732, 0.00012191893620183691, 0.00011536724196048453, 0.00010600918903946877, 9.877009142655879e-05, 9.469976066611707e-05, 9.101785690290853e-05, 8.574059756938368e-05, 8.224276098189875e-05, 7.74944419390522e-05, 7.435503357555717e-05, 7.267118780873716e-05, 7.040146010695025e-05, 6.70367298880592e-05, 6.324299465632066e-05, 6.0062087868573144e-05, 5.733635771321133e-05, 5.42059751751367e-05, 5.1946255553048104e-05, 5.0537077186163515e-05, 4.88759033032693e-05, 4.733542664325796e-05, 4.596458893502131e-05, 4.4207557948539034e-05, 4.3704218114726245e-05, 4.2863412090810016e-05, 4.1809489630395547e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002221399099653354, "first_to_last_ratio": 1.7678964788399845, "imbalance_abs_log_ratio": 0.5697904098088594, "last_quartile_mean": 0.00012565210272441618, "mean": 0.00016188397967198398, "normalized": [ 1.7325277794978797, 1.5951376332003655, 1.4520548687114199, 1.3729981481664384, 1.2857750985910605, 1.2257666436669268, 1.1762283345939986, 1.1372455678364286, 1.0889526987749056, 1.0509403055231545, 1.0259172538151122, 1.0049179037159137, 0.978085345877984, 0.9585059142501069, 0.9364886382982085, 0.9194669656082258, 0.9066971379228966, 0.893622399915035, 0.8769613699768213, 0.8605074489369314, 0.8461051105300481, 0.8336447776090111, 0.8211750960224249, 0.8107883690330443, 0.8025267555268564, 0.7926031470462279, 0.7851370690218832, 0.7775583575558719, 0.7689758331264998, 0.7655660871494063, 0.7607136802742225, 0.75640826022469 ], "peak_layer": 1, "peak_normalized": 1.7325277794978797, "population_cv": 0.2510639592948361, "rest_mean": 0.00016719473286583606, "spike_contrast": 0.796711177092359, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013320591242518275, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002804684918373823, 0.0002582272281870246, 0.00023506442084908485, 0.00022226640430744737, 0.00020814638992305845, 0.00019843198242597282, 0.00019041252380702645, 0.00018410183838568628, 0.00017628399655222893, 0.00017013039905577898, 0.00016607956786174327, 0.00016268010949715972, 0.00015833634824957699, 0.00015516675193794072, 0.00015160250768531114, 0.00014884697156958282, 0.00014677974104415625, 0.00014466315042227507, 0.00014196599659044296, 0.000139302370371297, 0.00013697086251340806, 0.00013495373423211277, 0.00013293509255163372, 0.00013125364785082638, 0.00012991622497793287, 0.00012830975174438208, 0.00012710111332125962, 0.000125874241348356, 0.00012448486813809723, 0.00012393288488965482, 0.00012314735795371234, 0.00012245037942193449 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19699589163064957, "first_to_last_ratio": 1.7678964443273588, "imbalance_abs_log_ratio": 0.5697903902870003, "last_quartile_mean": 0.11142954230308533, "mean": 0.1435603315476328, "normalized": [ 1.7325277292733527, 1.595137596567507, 1.4520549137362593, 1.3729980837787108, 1.285775112498371, 1.2257666266551688, 1.1762283724960378, 1.1372455675264566, 1.0889526722354506, 1.0509403711345948, 1.0259172522938773, 1.00491793591038, 0.9780853203912144, 0.9585059497750074, 0.9364885904027979, 0.9194669905174069, 0.9066971317518798, 0.8936224205946474, 0.8769613084029552, 0.8605074273314671, 0.84610509802098, 0.833644762521393, 0.821175085273337, 0.8107884103394608, 0.802526775587875, 0.7926031918850489, 0.7851370589142022, 0.7775583578743038, 0.7689758340628073, 0.7655660958716172, 0.7607136803251955, 0.7564082760502372 ], "peak_layer": 1, "peak_normalized": 1.7325277292733527, "population_cv": 0.2510639513231401, "rest_mean": 0.14826996044980156, "spike_contrast": 0.7967111822081809, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11812833547592164, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24872225522994995, 0.22899848222732544, 0.20845748484134674, 0.19710806012153625, 0.184586301445961, 0.17597146332263947, 0.16885973513126373, 0.16326335072517395, 0.156330406665802, 0.1508733481168747, 0.14728102087974548, 0.14426635205745697, 0.1404142528772354, 0.13760343194007874, 0.13444261252880096, 0.13199898600578308, 0.13016574084758759, 0.12828873097896576, 0.1258968561887741, 0.12353473156690598, 0.12146712839603424, 0.1196783185005188, 0.11788816750049591, 0.11639705300331116, 0.11521100997924805, 0.11378637701272964, 0.11271453648805618, 0.111626535654068, 0.1103944256901741, 0.10990492254495621, 0.10920830816030502, 0.10859022289514542 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.813338586653117e-05, "first_to_last_ratio": 2.039661655563216, "imbalance_abs_log_ratio": 0.7127839389845736, "last_quartile_mean": 3.3404258829250466e-05, "mean": 4.817700425974181e-05, "normalized": [ 1.6677237328958603, 1.5880861584886787, 1.4689677920146058, 1.4140322553861728, 1.3674596064698026, 1.3072824267629832, 1.2642946708581568, 1.2359968662296865, 1.1645552208108763, 1.101791124796405, 1.0797798333773554, 1.0672664620429624, 1.012803710836977, 0.9955853656459409, 0.9623936542224847, 0.9380946280712082, 0.9310631059210642, 0.9082494158597965, 0.8812030087749774, 0.8644086677601183, 0.8497891698936241, 0.8309347651333899, 0.7889921814106587, 0.7623244708429415, 0.7584228017816982, 0.7445258855646851, 0.7180452956289476, 0.7000407870401953, 0.6663749309645193, 0.6665299586713209, 0.6558653663654831, 0.637116679476424 ], "peak_layer": 1, "peak_normalized": 1.6677237328958603, "population_cv": 0.28258618188430573, "rest_mean": 4.9978354245131074e-05, "spike_contrast": 0.7693273401931907, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.844971433863975e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.034593338379636e-05, 7.650923362234607e-05, 7.077046757331118e-05, 6.812383799115196e-05, 6.588010728592053e-05, 6.298095104284585e-05, 6.090992974350229e-05, 5.9546626289375126e-05, 5.610478183371015e-05, 5.3080995712662116e-05, 5.2020557632204145e-05, 5.141770088812336e-05, 4.879384869127534e-05, 4.79643204016611e-05, 4.636524317902513e-05, 4.519458889262751e-05, 4.485583122004755e-05, 4.375673597678542e-05, 4.245372110744938e-05, 4.164462006883696e-05, 4.094029645784758e-05, 4.0031947719398886e-05, 3.8011279684724286e-05, 3.672650927910581e-05, 3.653853855212219e-05, 3.5869026760337874e-05, 3.4593271266203374e-05, 3.3725867979228497e-05, 3.21039478876628e-05, 3.211141665815376e-05, 3.1597628549207e-05, 3.0694372981088236e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018498812278266996, "first_to_last_ratio": 1.477115588497286, "imbalance_abs_log_ratio": 0.3900912594562188, "last_quartile_mean": 0.00012523605073511135, "mean": 0.00015057817927299766, "normalized": [ 1.364889512785021, 1.3222437098497068, 1.2702392636002102, 1.235819093505974, 1.2002761568067795, 1.1699209662016437, 1.143160310272995, 1.1216013166366416, 1.0931712967611817, 1.0676161315047876, 1.0503356923443616, 1.0352076251778792, 1.014929888181958, 1.0014649033450271, 0.9840777733314531, 0.96959004456439, 0.958410698647572, 0.9467186761221033, 0.9322053376846741, 0.9180751779000818, 0.904837007766356, 0.8925970349353171, 0.879899470446981, 0.8691033998232655, 0.8603564893899045, 0.8496764837250937, 0.8415817018035472, 0.8332582690137502, 0.8239257185712614, 0.820107268399535, 0.8147646079598274, 0.8099389729407196 ], "peak_layer": 1, "peak_normalized": 1.364889512785021, "population_cv": 0.1566386619724503, "rest_mean": 0.000153886474436149, "spike_contrast": 0.8624109810706347, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013271338539198042, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002055225777439773, 0.00019910105038434267, 0.000191270315553993, 0.00018608738901093602, 0.0001807353983167559, 0.00017616456898394972, 0.00017213499813806266, 0.00016888868412934244, 0.00016460774349980056, 0.00016075969324447215, 0.0001581576361786574, 0.00015587967936880887, 0.00015282629465218633, 0.00015079876175150275, 0.0001481806393712759, 0.00014599910355173051, 0.00014431573799811304, 0.00014255517453420907, 0.00014036978245712817, 0.00013824208872392774, 0.00013624870916828513, 0.00013440563634503633, 0.0001324936602031812, 0.00013086800754535943, 0.00012955091369803995, 0.00012794273789040744, 0.00012672384036704898, 0.0001254705130122602, 0.0001240652345586568, 0.00012349025928415358, 0.00012268577120266855, 0.00012195913586765528 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015959453048708383, "first_to_last_ratio": 1.2728956423420892, "imbalance_abs_log_ratio": 0.24129433847892473, "last_quartile_mean": 0.00012537911607068963, "mean": 0.00014144417764327955, "normalized": [ 1.1740787416792278, 1.1612764386268224, 1.1471117856731012, 1.1325725430810556, 1.1221203501929577, 1.1091452070987, 1.0952451640185767, 1.0850231158766837, 1.0710066115302646, 1.0558964823952193, 1.0428503514266292, 1.0346153464801762, 1.0229188081717178, 1.0151986196356395, 1.003700332970425, 0.9946300373693288, 0.9863513080075261, 0.9777764871930738, 0.967654233599155, 0.9595434059922948, 0.9513191006674802, 0.9429489101168639, 0.9334504238882736, 0.9221966853060849, 0.914156536930625, 0.9038869577772225, 0.8952163547729103, 0.8871546013922188, 0.8776639341179028, 0.8750860454698683, 0.8708633986378481, 0.8673416799041263 ], "peak_layer": 1, "peak_normalized": 1.1740787416792278, "population_cv": 0.09275162400716915, "rest_mean": 0.00014320399646681767, "spike_contrast": 0.9213510743517984, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001319411559961736, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001660666021052748, 0.00016425579087808728, 0.00016225228318944573, 0.0001601957919774577, 0.00015871739014983177, 0.0001568821317050606, 0.0001549160515423864, 0.00015347020234912634, 0.00015148764941841364, 0.0001493504096288234, 0.00014750511036254466, 0.00014634031686000526, 0.00014468590961769223, 0.00014359393389895558, 0.00014196756819728762, 0.0001406846276950091, 0.00013951364962849766, 0.00013830079114995897, 0.0001368690573144704, 0.00013572182797361165, 0.000134558547870256, 0.00013337463315110654, 0.00013203112757764757, 0.00013043935177847743, 0.00012930211960338056, 0.00012784954742528498, 0.00012662314111366868, 0.00012548285303637385, 0.00012414045340847224, 0.00012377582606859505, 0.00012317855725996196, 0.00012268043064977974 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031298474641516805, "first_to_last_ratio": 2.209030513243648, "imbalance_abs_log_ratio": 0.7925537374904509, "last_quartile_mean": 0.00014168421148497146, "mean": 0.0002071673161481158, "normalized": [ 1.8097158340785349, 1.7497804411308835, 1.611244862997228, 1.5435020349400896, 1.4351553463811126, 1.3679605489917586, 1.3130823229189843, 1.255818245825708, 1.1871164694634437, 1.122716192977336, 1.0885067728511792, 1.0444622322416663, 0.990152969051365, 0.9605806666596922, 0.9222549756846732, 0.8800158737132093, 0.8615285136033911, 0.8400222785845712, 0.8115627552032852, 0.7845983398409414, 0.7619432914958448, 0.7435905859348785, 0.7280762324918081, 0.7153162915808863, 0.7064441934578637, 0.6963930075638303, 0.6905250334602637, 0.686293775915277, 0.6782364888338203, 0.6751963304984743, 0.6708841537879804, 0.6673229378400193 ], "peak_layer": 1, "peak_normalized": 1.8097158340785349, "population_cv": 0.3382701486201724, "rest_mean": 0.00021748447411314205, "spike_contrast": 0.6963929896815652, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015145466313697397, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003749139723367989, 0.00036249731783755124, 0.00033379727392457426, 0.0003197631740476936, 0.0002973172813653946, 0.00028339671553112566, 0.0002720277407206595, 0.0002601644955575466, 0.0002459317329339683, 0.00023259010049514472, 0.0002255030267406255, 0.00021637843747157604, 0.00020512733317445964, 0.00019900091865565628, 0.00019106108811683953, 0.00018231052672490478, 0.00017848054994829, 0.00017402516095899045, 0.00016812927788123488, 0.0001625431323191151, 0.00015784974675625563, 0.00015404766600113362, 0.00015083359903655946, 0.00014819015632383525, 0.00014635214756708592, 0.00014426987036131322, 0.00014305421791505069, 0.00014217763964552432, 0.00014050843310542405, 0.0001398786116624251, 0.00013898526958655566, 0.0001382475020363927 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034565660171210766, 0.004928664769977331, 0.0060537648387253284, 0.006960673723369837, 0.0036145171616226435, 0.0051584807224571705, 0.006370538379997015, 0.007546419743448496, 0.00395671371370554, 0.005665866658091545, 0.006755253300070763, 0.007946331985294819, 0.004383862018585205, 0.006652849726378918, 0.007932198233902454, 0.009347881190478802, 0.004880861379206181, 0.006906712893396616, 0.008132148534059525, 0.009816600009799004, 0.005453306715935469, 0.007622495759278536, 0.009448722936213017, 0.010670131072402, 0.0055701653473079205, 0.007738742511719465, 0.009509840048849583, 0.010865328833460808, 0.005987230688333511, 0.00815181527286768, 0.009874933399260044, 0.011666664853692055 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015672678182454547, "first_to_last_ratio": 3.4451487067191406, "imbalance_abs_log_ratio": 1.236967069490208, "last_quartile_mean": 4.549202230919036e-05, "mean": 8.812991916329338e-05, "normalized": [ 2.495342024881723, 2.231858330120814, 1.9399329049931513, 1.7866550124027707, 1.6079479837593746, 1.479673880937976, 1.3811985381554308, 1.3042766606266647, 1.1954508396552082, 1.1168862767589318, 1.0692251193363627, 1.0303878333212484, 0.9676417435164101, 0.9260651349655372, 0.8738635380560305, 0.8395483633300559, 0.8220975248023003, 0.7970985234813915, 0.759112342831592, 0.7162079306537569, 0.6791607399033079, 0.6489075313744581, 0.615087961024076, 0.5868315353418163, 0.5699679008989185, 0.5521854623420828, 0.5335941062986479, 0.5198929115704046, 0.5024718770039331, 0.49422901772262035, 0.4850259920431499, 0.47217445788985424 ], "peak_layer": 1, "peak_normalized": 2.495342024881723, "population_cv": 0.5254832958744828, "rest_mean": 9.433179891655325e-05, "spike_contrast": 0.5792295824234721, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.4639768495690075e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021991429093759507, 0.0001966934942174703, 0.00017096613009925932, 0.0001574577618157491, 0.00014170832582749426, 0.00013040353951510042, 0.0001217249155160971, 0.00011494579666759819, 0.00010535498586250469, 9.843109728535637e-05, 9.423072333447635e-05, 9.080799645744264e-05, 8.527818863512948e-05, 8.161404548445717e-05, 7.701352296862751e-05, 7.398932939395308e-05, 7.245138840517029e-05, 7.024822843959555e-05, 6.690050940960646e-05, 6.311934703262523e-05, 5.985438110656105e-05, 5.7188168284483254e-05, 5.420765228336677e-05, 5.1717415772145614e-05, 5.02312250318937e-05, 4.866406015935354e-05, 4.702560545410961e-05, 4.581812027026899e-05, 4.428280590218492e-05, 4.3556363380048424e-05, 4.2745301470858976e-05, 4.161269680480473e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00022039370378479362, "first_to_last_ratio": 1.7624457042066763, "imbalance_abs_log_ratio": 0.5667024491058967, "last_quartile_mean": 0.00012504992537287762, "mean": 0.0001607882525149762, "normalized": [ 1.7301666786506156, 1.592657504255016, 1.4498087081325215, 1.3715848200709229, 1.2853506396348091, 1.224257634451794, 1.175979576607571, 1.1358564358860717, 1.0864628307036615, 1.050035766385231, 1.0256357250890054, 1.0049217142965674, 0.9775345112735686, 0.9578189361646354, 0.9358080993289649, 0.9184597363064851, 0.9072850761620224, 0.8934833681037547, 0.8779253472396409, 0.8613572157279928, 0.8467159963027754, 0.8352601414820391, 0.8224708066612338, 0.81131886291965, 0.8015955184587156, 0.7940537632065271, 0.7857694259399116, 0.7781749070905981, 0.7724085609032495, 0.7676732316943716, 0.7637136993200662, 0.75845476155001 ], "peak_layer": 1, "peak_normalized": 1.7301666786506156, "population_cv": 0.2498932502086342, "rest_mean": 0.0001660444721454513, "spike_contrast": 0.7974048446155259, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001324046665104106, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027819047681987286, 0.0002560806169640273, 0.0002331122086616233, 0.00022053472639527172, 0.00020666928321588784, 0.00019684624567162246, 0.0001890837011160329, 0.00018263237143401057, 0.00017469045997131616, 0.0001688334159553051, 0.0001649101759539917, 0.00016157960635609925, 0.00015717606584075838, 0.00015400603297166526, 0.00015046694898046553, 0.00014767753600608557, 0.00014588078192900866, 0.00014366162940859795, 0.00014116008242126554, 0.00013849612150806934, 0.0001361419854220003, 0.00013430001854430884, 0.00013224364374764264, 0.00013045054220128804, 0.00012888714263681322, 0.00012767451698891819, 0.0001263424928765744, 0.00012512138346210122, 0.00012419422273524106, 0.00012343283742666245, 0.0001227961911354214, 0.00012195061572128907 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19544734060764313, "first_to_last_ratio": 1.7624457434404193, "imbalance_abs_log_ratio": 0.5667024713668619, "last_quartile_mean": 0.11089552193880081, "mean": 0.14258862822316587, "normalized": [ 1.7301667593069614, 1.5926575752547714, 1.4498087309457477, 1.3715847721481056, 1.2853506436856033, 1.2242575941276408, 1.1759796246815233, 1.1358564592049036, 1.0864627991256137, 1.0500357507634608, 1.0256357136610779, 1.0049216567312906, 0.9775345659226533, 0.957818945514909, 0.935808091557486, 0.9184597083380731, 0.9072850362319335, 0.8934833286128027, 0.8779253174360568, 0.8613572174793438, 0.8467160247668938, 0.8352601831545204, 0.8224708111632975, 0.8113188687979516, 0.8015955037045681, 0.7940537757091718, 0.7857693860910103, 0.7781748832951502, 0.7724085309387246, 0.7676732205256189, 0.7637136996793319, 0.7584548214438025 ], "peak_layer": 1, "peak_normalized": 1.7301667593069614, "population_cv": 0.24989326289316924, "rest_mean": 0.14724989594132812, "spike_contrast": 0.797404859232464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11741778254508972, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24670210480690002, 0.2270948588848114, 0.20672623813152313, 0.1955723911523819, 0.18327638506889343, 0.17456521093845367, 0.16768132150173187, 0.16196021437644958, 0.15491724014282227, 0.14972315728664398, 0.14624398946762085, 0.14329040050506592, 0.13938531279563904, 0.13657408952713013, 0.13343559205532074, 0.13096190989017487, 0.12936852872371674, 0.12740056216716766, 0.1251821666955948, 0.12281974405050278, 0.12073207646608353, 0.11909860372543335, 0.11727498471736908, 0.11568484455347061, 0.11429840326309204, 0.11322303861379623, 0.11204177886247635, 0.11095888912677765, 0.11013667285442352, 0.1094614714384079, 0.1088968887925148, 0.108147032558918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.812703486502869e-05, "first_to_last_ratio": 2.0429306771418236, "imbalance_abs_log_ratio": 0.7143853833664048, "last_quartile_mean": 3.334769780849456e-05, "mean": 4.813682039639389e-05, "normalized": [ 1.670694398734861, 1.5890602892970849, 1.4709330435001327, 1.4135870318536654, 1.3688033811669116, 1.3072210435152478, 1.2659107055367187, 1.236022743120565, 1.164719165633699, 1.10067041168383, 1.075093368589445, 1.0685491843186352, 1.0112041551575939, 0.9945774787180313, 0.9602353052595208, 0.9352302198346567, 0.9313930096398116, 0.9092161974531191, 0.8814894006709005, 0.8653525310040872, 0.8506115464001769, 0.8320776390213925, 0.7932455071661803, 0.7619500965894176, 0.7566697664554205, 0.7419550822306056, 0.7141621566216388, 0.7010675908642553, 0.6716466869044494, 0.66592144202409, 0.6557448579818498, 0.6349845630520059 ], "peak_layer": 1, "peak_normalized": 1.670694398734861, "population_cv": 0.28301355753944435, "rest_mean": 4.992937398061191e-05, "spike_contrast": 0.7702285844110489, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.845703104161657e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.042191620916128e-05, 7.649230974493548e-05, 7.080603973008692e-05, 6.804558506701142e-05, 6.588984251720831e-05, 6.292546459008008e-05, 6.0936916270293295e-05, 5.9498204791452736e-05, 5.606587728834711e-05, 5.2982773922849447e-05, 5.1751576393144205e-05, 5.143656017025933e-05, 4.867615280090831e-05, 4.7875797463348135e-05, 4.622267442755401e-05, 4.501900912146084e-05, 4.483429802348837e-05, 4.3766776798292994e-05, 4.243209696142003e-05, 4.1655319364508614e-05, 4.094573523616418e-05, 4.005357186542824e-05, 3.8184316508704796e-05, 3.667785495053977e-05, 3.6423676647245884e-05, 3.571535853552632e-05, 3.437749546719715e-05, 3.374716470716521e-05, 3.233093593735248e-05, 3.2055340852821246e-05, 3.156547245453112e-05, 3.056613786611706e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018337804431212135, "first_to_last_ratio": 1.471219679743085, "imbalance_abs_log_ratio": 0.38609177087911456, "last_quartile_mean": 0.00012464355040719965, "mean": 0.0001495256622092711, "normalized": [ 1.3620802882826015, 1.3194938018514704, 1.2677903290886852, 1.23356455122679, 1.198458119977671, 1.1684656874236388, 1.1415318441782234, 1.1198031889553155, 1.0910031274146845, 1.0660983196947673, 1.0498697336705507, 1.0350765279986567, 1.0145895852928877, 1.0013439705330982, 0.9836063325227735, 0.968549777523238, 0.9591067674489386, 0.9465820057822166, 0.9334010118478351, 0.9191551367815048, 0.9057362914957724, 0.8945632145340071, 0.8814992971854174, 0.8698868180749819, 0.8596244665411801, 0.8514889576556698, 0.8425072468470839, 0.8341767076391075, 0.8278204125037605, 0.8225904077486038, 0.8181934666462969, 0.8123426056325708 ], "peak_layer": 1, "peak_normalized": 1.3620802882826015, "population_cv": 0.1550560200772002, "rest_mean": 0.00015278581957789829, "spike_contrast": 0.8634362323882038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013192081241868435, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002036659570876509, 0.00019729818450286984, 0.0001895671884994954, 0.0001844495564000681, 0.00017920024401973933, 0.00017471560568083078, 0.00017068830493371934, 0.0001674393133725971, 0.0001631329650990665, 0.00015940905723255128, 0.0001569824671605602, 0.0001547705032862723, 0.00015170717961154878, 0.00014972662029322237, 0.00014707438822370023, 0.00014482304686680436, 0.00014341107453219593, 0.00014153830124996603, 0.00013956740440335125, 0.00013743728050030768, 0.00013543081877287477, 0.00013376015704125166, 0.0001318067661486566, 0.00013007040251977742, 0.00012853591761086136, 0.00012731945025734603, 0.00012597645400092006, 0.00012473082460928708, 0.00012378039536997676, 0.00012299837544560432, 0.0001223409199155867, 0.00012146606604801491 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015817073654034175, "first_to_last_ratio": 1.2671327323753674, "imbalance_abs_log_ratio": 0.23675665700060805, "last_quartile_mean": 0.00012482570491556544, "mean": 0.00014045925900063594, "normalized": [ 1.1719218726461313, 1.1599555891510773, 1.143372783062998, 1.1305925991754215, 1.1202702784385838, 1.106568447670676, 1.0931409560371117, 1.082952696795682, 1.0683580206124534, 1.05266391584401, 1.0427272032106563, 1.034492263470324, 1.0219716035750168, 1.0154196846947314, 1.0039021031599646, 0.9939448772529704, 0.9863124890252286, 0.9768583563343864, 0.9682383265025308, 0.9606909958387225, 0.9520472410590707, 0.9456933063501335, 0.9351354973759991, 0.9231938712502704, 0.9140830768869257, 0.9051693342733999, 0.8960380266364146, 0.8881305740575848, 0.8822109405764296, 0.8783208779569746, 0.8754534745501711, 0.8701687165279489 ], "peak_layer": 1, "peak_normalized": 1.1719218726461313, "population_cv": 0.09095723041158388, "rest_mean": 0.00014217518737832843, "spike_contrast": 0.9227576779061385, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013119324576109647, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016460727783851326, 0.00016292650252580643, 0.00016059729387052357, 0.0001588021987117827, 0.0001573523331899196, 0.00015542778419330716, 0.00015354176866821945, 0.00015211073332466185, 0.00015006077592261136, 0.00014785639359615743, 0.0001464606903027743, 0.00014530401676893234, 0.00014354537415783852, 0.00014262509648688138, 0.0001410073455190286, 0.00013960876094643027, 0.00013853672135155648, 0.0001372088008793071, 0.00013599803787656128, 0.00013493794540409, 0.00013372385001275688, 0.00013283138105180115, 0.00013134843902662396, 0.0001296711270697415, 0.0001283914316445589, 0.00012713941396214068, 0.00012585683725774288, 0.00012474616232793778, 0.00012391469499561936, 0.00012336829968262464, 0.00012296554632484913, 0.00012222325312905014 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031102553475648165, "first_to_last_ratio": 2.2098501973960105, "imbalance_abs_log_ratio": 0.7929247292485178, "last_quartile_mean": 0.00014074507635086775, "mean": 0.00020578417843353236, "normalized": [ 1.8085758305333837, 1.7408578335592815, 1.6197863489261306, 1.5416881776712252, 1.4477928223214513, 1.3679195769143782, 1.3100342829404608, 1.2546744848740619, 1.192398672058626, 1.1190508147725688, 1.082828610043334, 1.0467691899805913, 0.9849335746234156, 0.9618381641711736, 0.9182125099812896, 0.877817233347749, 0.8611427657722097, 0.8409392227987704, 0.8149468534272446, 0.7879793317746896, 0.7604794717340572, 0.7439065501103351, 0.7274095758071225, 0.7164574630456112, 0.7035124756224637, 0.6971408207181957, 0.690610058298894, 0.684463275349207, 0.6799680284055707, 0.6760380727842366, 0.6720531720343774, 0.6677747355978938 ], "peak_layer": 1, "peak_normalized": 1.8085758305333837, "population_cv": 0.33829873005197686, "rest_mean": 0.0002160599310796156, "spike_contrast": 0.6956177084463702, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015029511414468288, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003721762914210558, 0.00035824099904857576, 0.00033332640305161476, 0.00031725503504276276, 0.00029793285648338497, 0.0002814962062984705, 0.0002695843286346644, 0.00025819215807132423, 0.0002453767810948193, 0.00023028295254334807, 0.00022282899590209126, 0.00021540853776969016, 0.00020268374646548182, 0.00019793107639998198, 0.0001889536069938913, 0.00018064089817926288, 0.00017720955656841397, 0.0001730519870761782, 0.00016770316869951785, 0.00016215367941185832, 0.00015649464330635965, 0.00015308419824577868, 0.00014968938194215298, 0.00014743561041541398, 0.00014477173681370914, 0.00014346055104397237, 0.00014211662346497178, 0.00014085171278566122, 0.00013992666208650917, 0.00013911793939769268, 0.00013829790987074375, 0.00013741747534368187 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.028097106143832207, 0.014136734418570995, 0.014224153943359852, 0.014337832108139992, 0.014408928342163563, 0.009704872034490108, 0.009773381985723972, 0.00987160298973322, 0.00994801800698042, 0.007515520788729191, 0.007602147292345762, 0.007658426649868488, 0.007734050042927265, 0.0062688966281712055, 0.006336068734526634, 0.006399397272616625, 0.006462901830673218, 0.005432929843664169, 0.005504137370735407, 0.005560056772083044, 0.005631812382489443, 0.004873365629464388, 0.004937449004501104, 0.005005750339478254, 0.005022916477173567, 0.0044684382155537605, 0.004500201437622309, 0.004573327023535967, 0.00463611725717783, 0.004166214261204004, 0.0042496914975345135, 0.004283483140170574 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00014323742470878642, "first_to_last_ratio": 1.6208859606473658, "imbalance_abs_log_ratio": 0.48297288904334346, "last_quartile_mean": 8.83698348843609e-05, "mean": 0.00011961164659624046, "normalized": [ 0.879665217639624, 1.4482702231163533, 1.4522235545753037, 1.2017346580032677, 1.034069363337856, 1.36550769437152, 1.149442648944982, 1.0492523703464898, 1.0286339735981567, 1.1048881958426662, 1.2960670227766034, 1.1342833655748923, 0.9483258681651744, 1.1110988010941807, 1.0051223270547462, 0.9186295907129635, 0.9340104771989933, 1.0935042558126027, 0.9904421395235656, 1.070541841916413, 0.9831461473937306, 1.1165574278336847, 0.8906099662074067, 0.8835226812669187, 0.8406995671395937, 0.8651479324264472, 0.7705397043177317, 0.6783637574582257, 0.6658814129635636, 0.7912689330389562, 0.6809428819470401, 0.6176059984003462 ], "peak_layer": 3, "peak_normalized": 1.4522235545753037, "population_cv": 0.21063690333645962, "rest_mean": 0.00012087627045629133, "spike_contrast": 0.9330423359872595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011278267775196582, "top_five_layers": [ 3, 2, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010521820513531566, 0.00017322998610325158, 0.00017370285058859736, 0.00014374146121554077, 0.000123686739243567, 0.0001633306237636134, 0.00013748672790825367, 0.00012550280371215194, 0.00012303660332690924, 0.0001321574964094907, 0.00015502471069339663, 0.00013567350106313825, 0.00011343081860104576, 0.0001329003571299836, 0.00012022433656966314, 0.00010987879795720801, 0.0001117185311159119, 0.00013079584459774196, 0.00011846841516671702, 0.00012804927246179432, 0.00011759572953451425, 0.00013355327246244997, 0.00010652732453309, 0.00010567960271146148, 0.00010055745951831341, 0.00010348176874686033, 9.216552280122414e-05, 8.114020602079108e-05, 7.964717224240303e-05, 9.464497998123989e-05, 8.144869934767485e-05, 7.387287041638047e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00015447182067873655, "first_to_last_ratio": 1.6506746305835311, "imbalance_abs_log_ratio": 0.5011840713694677, "last_quartile_mean": 9.358102306578076e-05, "mean": 0.00012785212516064348, "normalized": [ 0.8854544129111614, 1.4719853585239924, 1.4439160513495162, 1.202741955390318, 1.0425045283644727, 1.3767992039076578, 1.1738833181051227, 1.0683704265103389, 1.0407410271769484, 1.136073438345882, 1.2855554885667506, 1.129314905977533, 0.953735223493217, 1.1222184469907663, 1.0164965849441758, 0.9298350247961144, 0.9320412792829462, 1.0837813144211186, 0.9857736034320764, 1.0482233302985868, 0.9656597421723547, 1.0923472818519362, 0.884471876187184, 0.8724974481708365, 0.826330173068434, 0.8536374096219558, 0.7648976975162065, 0.681485935203229, 0.6604246501739865, 0.7781176964145518, 0.6752223987471391, 0.6154627680834903 ], "peak_layer": 2, "peak_normalized": 1.4719853585239924, "population_cv": 0.21455917947613024, "rest_mean": 0.00012955063343893185, "spike_contrast": 0.9160910858365683, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011868018045788631, "top_five_layers": [ 2, 3, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001132072284235619, 0.00018819645629264414, 0.00018460773571860045, 0.00015377311501652002, 0.00013328641944099218, 0.00017602670413907617, 0.0001500834769103676, 0.00013659342948812991, 0.00013306095206644386, 0.00014524940343108028, 0.00016436100122518837, 0.00014438531070481986, 0.00012193707516416907, 0.0001434780133422464, 0.00012996124860364944, 0.00011888138396898285, 0.00011916345829376951, 0.00013856374425813556, 0.00012603325012605637, 0.00013401758042164147, 0.0001234616502188146, 0.00013965892139822245, 0.00011308160901535302, 0.00011155065294587985, 0.00010564806871116161, 0.00010913935693679377, 9.779379615793005e-05, 8.71294250828214e-05, 8.443669503321871e-05, 9.948400111170486e-05, 8.632861863588914e-05, 7.86882228567265e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.13698715530335903, "first_to_last_ratio": 1.6506746463483835, "imbalance_abs_log_ratio": 0.5011840809200188, "last_quartile_mean": 0.08298858627676964, "mean": 0.11338054295629263, "normalized": [ 0.8854544185536968, 1.4719853260144444, 1.4439159993116888, 1.2027420311824657, 1.042504519270847, 1.3767991933658148, 1.1738833593963587, 1.0683704218985235, 1.0407409783116481, 1.1360735196220915, 1.285555449073044, 1.129314933056767, 0.9537352653592515, 1.1222184500196979, 1.0164965450422543, 0.9298350370138166, 0.9320412867497836, 1.0837813135283418, 0.9857735942006465, 1.0482233283425098, 0.9656597514448471, 1.0923472716531482, 0.8844718771212735, 0.8724974491223123, 0.82633015871154, 0.8536374112441639, 0.7648976624892803, 0.681485927333974, 0.660424633911195, 0.7781176812354046, 0.6752224228414072, 0.6154627835777616 ], "peak_layer": 2, "peak_normalized": 1.4719853260144444, "population_cv": 0.2145591770090877, "rest_mean": 0.11488679713673061, "spike_contrast": 0.9160910827435619, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10524677038192749, "top_five_layers": [ 2, 3, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.10039330273866653, 0.16689449548721313, 0.16371197998523712, 0.1363675445318222, 0.11819972842931747, 0.1561022400856018, 0.13309553265571594, 0.12113241851329803, 0.11799977719783783, 0.1288086324930191, 0.14575697481632233, 0.1280423402786255, 0.10813502222299576, 0.1272377371788025, 0.11525093019008636, 0.10542520135641098, 0.1056753471493721, 0.12287971377372742, 0.11176754534244537, 0.11884813010692596, 0.10948702692985535, 0.12385092675685883, 0.10028190165758133, 0.09892423450946808, 0.09368976205587387, 0.09678587317466736, 0.08672451227903366, 0.07726724445819855, 0.07487930357456207, 0.0882234051823616, 0.07655708491802216, 0.06978150457143784 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.7160250005617854e-05, "first_to_last_ratio": 0.7388534944946782, "imbalance_abs_log_ratio": 0.30265562600345663, "last_quartile_mean": 6.382895981005277e-05, "mean": 5.960399772675373e-05, "normalized": [ 0.46158176249046556, 0.8375260596839308, 0.8768757910510345, 0.7241656982213429, 0.6710453705877814, 1.0089930699998053, 0.8842011269201543, 0.865421443073546, 0.883514351886623, 0.9622394496647411, 1.1256912972870226, 1.0210990974149952, 0.9232791878576234, 1.0980011773561495, 1.0212452171652862, 0.9544893655323485, 1.0133393696232265, 1.159814348379829, 0.9921993085906312, 1.1469178460749072, 1.0774269793999733, 1.3924548133021735, 1.1792402184404436, 1.1521666823514416, 1.1069955376913123, 1.1868921569761495, 1.1034570471301723, 0.9847635589252388, 0.9837749617675684, 1.1772881708824627, 1.0498861530819708, 0.9740133811896496 ], "peak_layer": 22, "peak_normalized": 1.3924548133021735, "population_cv": 0.17243044689052606, "rest_mean": 5.759890990264938e-05, "spike_contrast": 1.22279175241262, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.043147197691724e-05, "top_five_layers": [ 22, 26, 23, 30, 18 ], "top_five_spike_overlap": 2 }, "values": [ 2.751211832219269e-05, 4.991990135749802e-05, 5.226530265645124e-05, 4.316317063057795e-05, 3.9996986743062735e-05, 6.014002065057866e-05, 5.2701921958941966e-05, 5.158257772563957e-05, 5.266098742140457e-05, 5.735331797040999e-05, 6.709570152452216e-05, 6.0861588281113654e-05, 5.503113061422482e-05, 6.544525967910886e-05, 6.087029760237783e-05, 5.689138197340071e-05, 6.0399077483452857e-05, 6.912957178428769e-05, 5.9139045333722606e-05, 6.836088869022205e-05, 6.421895523089916e-05, 8.299587352667004e-05, 7.028743129922077e-05, 6.867374031571671e-05, 6.598135951207951e-05, 7.074351742630824e-05, 6.577045132871717e-05, 5.869584492756985e-05, 5.863692058483139e-05, 7.017108146101236e-05, 6.2577411881648e-05, 5.8055091358255595e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6127787956938846e-05, "first_to_last_ratio": 0.8333328618421979, "imbalance_abs_log_ratio": 0.18232212258347721, "last_quartile_mean": 6.735338365615462e-05, "mean": 6.689512088087213e-05, "normalized": [ 0.49022256384162527, 0.923222562655389, 0.9170289550360846, 0.7742286879065733, 0.7345865645537341, 1.0394905128518102, 0.9423822415780315, 0.8911705562121757, 0.9138181637864031, 1.0330628394205372, 1.1540375572503838, 1.0455389170050469, 0.9297420351318767, 1.1341544713967917, 1.0374336268991433, 0.9867016024211971, 1.0073449670971224, 1.1743962796261593, 1.0459325434797282, 1.1475080782429012, 1.066139536390961, 1.3297286391604237, 1.1245973641634917, 1.1027270030076322, 1.04892482254714, 1.125623033590473, 1.0476398534474196, 0.9361019469456514, 0.930671663614635, 1.0896207315037236, 0.9754527388499041, 0.9007689403858299 ], "peak_layer": 22, "peak_normalized": 1.3297286391604237, "population_cv": 0.14905429467389167, "rest_mean": 6.522988486621116e-05, "spike_contrast": 1.1633838617941636, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.58873953600414e-05, "top_five_layers": [ 22, 18, 11, 20, 14 ], "top_five_spike_overlap": 1 }, "values": [ 3.2793497666716576e-05, 6.17590849287808e-05, 6.134476279839873e-05, 5.179212166694924e-05, 4.9140257033286616e-05, 6.953684351174161e-05, 6.304077396634966e-05, 5.961496208328754e-05, 6.112997652962804e-05, 6.910686352057382e-05, 7.719948189333081e-05, 6.994145223870873e-05, 6.219520582817495e-05, 7.586940046167001e-05, 6.939924787729979e-05, 6.600552296731621e-05, 6.738646334270015e-05, 7.856138108763844e-05, 6.996778392931446e-05, 7.676269160583615e-05, 7.13195331627503e-05, 8.895235805539414e-05, 7.523007661802694e-05, 7.37670561647974e-05, 7.016795279923826e-05, 7.529868889832869e-05, 7.00819946359843e-05, 6.26206528977491e-05, 6.225739343790337e-05, 7.289031054824591e-05, 6.525302887894213e-05, 6.025704715284519e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.866022898866504e-05, "first_to_last_ratio": 0.40574519905788725, "imbalance_abs_log_ratio": 0.9020299049238899, "last_quartile_mean": 4.599001795213553e-05, "mean": 3.414976504245715e-05, "normalized": [ 0.2445526901503676, 0.5596630531980494, 0.5376964344799271, 0.48012169469058175, 0.4798804572179837, 0.6900344095639286, 0.7152325131545565, 0.6642055410913166, 0.7554442803991458, 0.9343372865664744, 0.9240467958883766, 0.8592444663674569, 0.8263415713413451, 1.0773994704320806, 0.959597390691018, 0.9894020700335048, 0.9760686528076964, 1.1936439731862338, 1.099336047654388, 1.1448507248259905, 1.0495120137852647, 1.4542665199347329, 1.3223844598378258, 1.2890134714336634, 1.2402412095084396, 1.4030995672846114, 1.3877316376933018, 1.246635573851661, 1.2962845787221184, 1.4847061188341568, 1.3923191445524272, 1.3227061808213745 ], "peak_layer": 30, "peak_normalized": 1.4847061188341568, "population_cv": 0.32662955778396485, "rest_mean": 3.2435424482510255e-05, "spike_contrast": 1.3382653305362568, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.340720406617038e-05, "top_five_layers": [ 30, 22, 26, 31, 27 ], "top_five_spike_overlap": 1 }, "values": [ 8.351416909135878e-06, 1.9112361769657582e-05, 1.8362206901656464e-05, 1.6396043065469712e-05, 1.6387804862461053e-05, 2.3564512957818806e-05, 2.4425022274954244e-05, 2.2682463168166578e-05, 2.5798244678298943e-05, 3.1907398806652054e-05, 3.1555980967823416e-05, 2.9342996640480123e-05, 2.8219370506121777e-05, 3.679293877212331e-05, 3.277002542745322e-05, 3.378784822416492e-05, 3.333251515869051e-05, 4.0762661228654906e-05, 3.7542067730100825e-05, 3.909638326149434e-05, 3.584058868000284e-05, 4.9662859964882955e-05, 4.515911859925836e-05, 4.4019507186021656e-05, 4.2353945900686085e-05, 4.791552055394277e-05, 4.7390709369210526e-05, 4.257231194060296e-05, 4.4267813791520894e-05, 5.070236511528492e-05, 4.754737165058032e-05, 4.517010529525578e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00021511870181711856, "first_to_last_ratio": 1.067203936185183, "imbalance_abs_log_ratio": 0.06504208450186474, "last_quartile_mean": 0.00020157225299044512, "mean": 0.00023711549874860793, "normalized": [ 0.5199215340336985, 0.9694171998286779, 0.9952311992308057, 0.8139231798698612, 0.8071491568326964, 1.1522101416966866, 1.0127543403881063, 0.9872470097646113, 0.9739878350012732, 1.0629045356723543, 1.2654986331851705, 1.158397891601516, 1.0251221075049282, 1.1837639471542862, 1.126357049236326, 1.0240452993420521, 1.0679809871494306, 1.2152134226291609, 1.0519957926786496, 1.2134701299884076, 1.1240302451392508, 1.3270417853241223, 1.0642149202553777, 1.0573092573282454, 0.9986311292560726, 1.01615500686031, 0.8954142860763721, 0.793725079326075, 0.7548043496765433, 0.8883375219775711, 0.764305251609244, 0.6894397743821165 ], "peak_layer": 22, "peak_normalized": 1.3270417853241223, "population_cv": 0.17546356070810173, "rest_mean": 0.00023209894856403548, "spike_contrast": 1.138328593816985, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002642048697452992, "top_five_layers": [ 22, 11, 18, 20, 14 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012328145385254174, 0.00022986384283285588, 0.0002359847421757877, 0.00019299380073789507, 0.00019138757488690317, 0.00027320688241161406, 0.00024013975053094327, 0.00023409156710840762, 0.00023094761127140373, 0.0002520311390981078, 0.00030006933957338333, 0.0002746740938164294, 0.00024307233979925513, 0.00028068877873010933, 0.0002670767134986818, 0.0002428170118946582, 0.00025323484442196786, 0.0002881459367927164, 0.00024944450706243515, 0.0002877325750887394, 0.0002665249921847135, 0.00031466217478737235, 0.0002523418515920639, 0.00025070441188290715, 0.0002367909182794392, 0.00024094610125757754, 0.00021231660502962768, 0.00018820451805368066, 0.00017897580983117223, 0.00021063859458081424, 0.00018122862093150616, 0.00016347685595974326 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014079716056585312, 0.014167090877890587, 0.014255538582801819, 0.014355218037962914, 0.009655896574258804, 0.009734091348946095, 0.009828771464526653, 0.009915035218000412, 0.007479202933609486, 0.007559895049780607, 0.0076358383521437645, 0.007706487085670233, 0.0062490664422512054, 0.006315518636256456, 0.0063744354993104935, 0.006439714692533016, 0.0054116579703986645, 0.005481995176523924, 0.005541813559830189, 0.005614723544567823, 0.004862982779741287, 0.004923288244754076, 0.0049925679340958595, 0.005004179198294878, 0.004448203835636377, 0.004490054678171873, 0.00455933902412653, 0.004625064320862293, 0.004160413984209299, 0.004242690280079842, 0.004270957782864571, 0.004303553607314825 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.6855086818832206e-05, "first_to_last_ratio": 1.1121706932103832, "imbalance_abs_log_ratio": 0.10631368513677814, "last_quartile_mean": 3.313797697046539e-05, "mean": 3.537216105087282e-05, "normalized": [ 1.3329674878497746, 1.065596392820356, 0.9206330800727252, 0.8842664165696186, 1.1462361065020723, 1.065158463293282, 0.99582829507555, 0.9247019781352203, 1.132583771771479, 1.005206548698436, 0.979948053013938, 0.967083333200755, 1.1057115808592233, 1.043785177992708, 0.995436955984306, 0.9279460300710988, 1.1098294348763655, 1.0288442525987258, 1.0234795644677463, 0.9449806453153974, 1.0109568163965834, 1.0260998393764513, 0.9416381672270038, 0.926379336673748, 0.9884731771330477, 0.9341156116917169, 0.9458688462068515, 0.9053302302997448, 0.9662542702797254, 0.9495526789453999, 0.9100381269887573, 0.8950693296121922 ], "peak_layer": 1, "peak_normalized": 1.3329674878497746, "population_cv": 0.09165679612351446, "rest_mean": 3.551162256001219e-05, "spike_contrast": 0.9748658722370791, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.461906890152022e-05, "top_five_layers": [ 1, 5, 9, 17, 13 ], "top_five_spike_overlap": 0 }, "values": [ 4.714994065579958e-05, 3.769244722207077e-05, 3.256478157709353e-05, 3.127841409877874e-05, 4.054484816151671e-05, 3.767695670831017e-05, 3.5224598832428455e-05, 3.270870729465969e-05, 4.006193557870574e-05, 3.555632792995311e-05, 3.4662880352698267e-05, 3.420782741159201e-05, 3.911140811396763e-05, 3.692093741847202e-05, 3.521075632306747e-05, 3.2823456422192976e-05, 3.9257065509445965e-05, 3.6392444599187e-05, 3.620268398663029e-05, 3.342600757605396e-05, 3.575972732505761e-05, 3.6295368772698566e-05, 3.330777690280229e-05, 3.276803909102455e-05, 3.49644324160181e-05, 3.3041687856893986e-05, 3.3457425161032006e-05, 3.202348671038635e-05, 3.417850166442804e-05, 3.358773028594442e-05, 3.2190015190280974e-05, 3.166053647873923e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.015105893515283e-05, "first_to_last_ratio": 0.6588274570259804, "imbalance_abs_log_ratio": 0.4172936042156455, "last_quartile_mean": 0.00010647865110513521, "mean": 9.056432463694364e-05, "normalized": [ 0.8598786252117567, 0.7452797309823858, 0.6694431442719111, 0.6417832925446045, 0.8822867975473095, 0.8318257491363829, 0.804532248804658, 0.7617644993607329, 0.969439786050785, 0.9144645011121496, 0.8951531228323205, 0.876468558935897, 1.0407786059783084, 1.0056397226462848, 0.9828357943952414, 0.9490839045924557, 1.1213662721801265, 1.0774120583919133, 1.0554308943167812, 1.0307906296953837, 1.1314055055223455, 1.1556700189426168, 1.1045075200854564, 1.0869669205578247, 1.2145839027446308, 1.1552275050200584, 1.1514380979421175, 1.1280886596086177, 1.2254687972249847, 1.20187962367999, 1.1639632986601018, 1.165142211023867 ], "peak_layer": 29, "peak_normalized": 1.2254687972249847, "population_cv": 0.1618495930711073, "rest_mean": 8.82393912804589e-05, "spike_contrast": 1.1686273359956587, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010311896476196125, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 7.787432696204633e-05, 6.74957555020228e-05, 6.062766624381766e-05, 5.812267045257613e-05, 7.990370795596391e-05, 7.533373718615621e-05, 7.286191976163536e-05, 6.898868741700426e-05, 8.779665949987248e-05, 8.281785994768143e-05, 8.106893801596016e-05, 7.937678310554475e-05, 9.425741154700518e-05, 9.107508230954409e-05, 8.900985994841903e-05, 8.595314284320921e-05, 0.00010155577911064029, 9.757509542396292e-05, 9.558438614476472e-05, 9.335285722045228e-05, 0.00010246497549815103, 0.00010466247476870194, 0.00010002897761296481, 9.844042506301776e-05, 0.00010999797086697072, 0.00010462239879416302, 0.00010427921370137483, 0.00010216458758804947, 0.00011098375398432836, 0.00010884741641348228, 0.00010541355004534125, 0.00010552031744737178 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06221066089347005, "first_to_last_ratio": 0.6588274447143223, "imbalance_abs_log_ratio": 0.4172936229028725, "last_quartile_mean": 0.09442633483558893, "mean": 0.0803133494919166, "normalized": [ 0.8598786809861813, 0.7452797356777653, 0.6694431594543391, 0.6417832788211475, 0.8822868217673959, 0.83182573854958, 0.804532202278358, 0.7617644434307961, 0.9694397651274825, 0.9144645475330049, 0.8951531251987148, 0.8764685422804359, 1.0407786734529123, 1.0056396718254574, 0.9828357789429103, 0.9490838578168909, 1.121366176654653, 1.0774119964231388, 1.0554309636293913, 1.030790617723408, 1.131405440813584, 1.1556700726993148, 1.1045075638748008, 1.0869669141870728, 1.214583886452764, 1.155227565075462, 1.151438141297865, 1.1280886745503538, 1.2254688318483915, 1.20187965027907, 1.1639633335063522, 1.1651421478410058 ], "peak_layer": 29, "peak_normalized": 1.2254688318483915, "population_cv": 0.16184959877781283, "rest_mean": 0.07825157529226055, "spike_contrast": 1.1686273385361958, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0914469301700592, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.06905973702669144, 0.05985591188073158, 0.05376522243022919, 0.051543764770030975, 0.0708594098687172, 0.06680671125650406, 0.0646146759390831, 0.06117985397577286, 0.07785895466804504, 0.0734437108039856, 0.07189274579286575, 0.07039212435483932, 0.08358842134475708, 0.08076629042625427, 0.07893483340740204, 0.07622410356998444, 0.09006067365407944, 0.08653056621551514, 0.08476519584655762, 0.08278624713420868, 0.09086696058511734, 0.09281573444604874, 0.08870670199394226, 0.0872979536652565, 0.09754730015993118, 0.09278019517660141, 0.09247585386037827, 0.09060057997703552, 0.09842150658369064, 0.09652698040008545, 0.09348179399967194, 0.09357646852731705 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.077007694533677e-06, "first_to_last_ratio": 0.4632676420751026, "imbalance_abs_log_ratio": 0.7694503312712233, "last_quartile_mean": 1.5276283193088602e-05, "mean": 1.1745167029175718e-05, "normalized": [ 0.5860812018784138, 0.5338863181232825, 0.4896831424227033, 0.48432444022666415, 0.7227742595277804, 0.658356157633798, 0.6679626701195096, 0.6773027265164295, 0.9317627933811012, 0.8396840560424388, 0.8002283441953012, 0.8237560838623014, 0.9947731150675175, 0.9485376018841178, 1.012754293639493, 0.980582798985168, 1.1861357451339187, 1.074396404738039, 1.1583203932505854, 1.1590437197037688, 1.1716087388237089, 1.3380122311407743, 1.182136735603702, 1.17274255168395, 1.3256589211015541, 1.1785808128912905, 1.3283612705891032, 1.2366027296837898, 1.3388375403515687, 1.3849270873253279, 1.3156503626951777, 1.2965347517777213 ], "peak_layer": 30, "peak_normalized": 1.3849270873253279, "population_cv": 0.2782190894097951, "rest_mean": 1.1227440423950881e-05, "spike_contrast": 1.2951207174852213, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4540890697389841e-05, "top_five_layers": [ 30, 29, 22, 27, 25 ], "top_five_spike_overlap": 2 }, "values": [ 6.883621608722024e-06, 6.2705839809495956e-06, 5.751410299126292e-06, 5.6884714467742015e-06, 8.48910440254258e-06, 7.732503036095295e-06, 7.845333129807841e-06, 7.955033652251586e-06, 1.0943709639832377e-05, 9.862229489954188e-06, 9.39881556405453e-06, 9.67515279626241e-06, 1.168377639260143e-05, 1.1140732567582745e-05, 1.1894968338310719e-05, 1.1517108760017436e-05, 1.3931362445873674e-05, 1.2618965229194146e-05, 1.3604666492028628e-05, 1.3613162082037888e-05, 1.3760740330326371e-05, 1.5715177141828462e-05, 1.3884393410990015e-05, 1.377405715174973e-05, 1.5570085452054627e-05, 1.38426285047899e-05, 1.56018249981571e-05, 1.4524105608870741e-05, 1.572487053635996e-05, 1.62661999638658e-05, 1.5452533261850476e-05, 1.5228017218760215e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.028231200616574e-05, "first_to_last_ratio": 0.5725175422355306, "imbalance_abs_log_ratio": 0.5577119025637401, "last_quartile_mean": 0.00010529338851483772, "mean": 8.693937525094952e-05, "normalized": [ 0.6520176082954547, 0.61792639701702, 0.5860559002851884, 0.5711155588169554, 0.8204737244004536, 0.784378974164202, 0.772878878140371, 0.7422194596148288, 0.9603819210570083, 0.917062379210463, 0.9022792075902993, 0.8821977885039085, 1.057360487646653, 1.0255127545988867, 1.0067830966339102, 0.9749202156934387, 1.1510091959216098, 1.1091409226242501, 1.0872456061539109, 1.0647793564016073, 1.1683829082644246, 1.1939706281941995, 1.1404893610428462, 1.1225154239128359, 1.2535795945499137, 1.1915479696415308, 1.1872717446378291, 1.1619194447200984, 1.2611193116864508, 1.2370940006457103, 1.1981927894482458, 1.1981773904854955 ], "peak_layer": 29, "peak_normalized": 1.2611193116864508, "population_cv": 0.20676778332368514, "rest_mean": 8.410921549916061e-05, "spike_contrast": 1.215351222858925, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001022222379106097, "top_five_layers": [ 29, 25, 30, 31, 32 ], "top_five_spike_overlap": 1 }, "values": [ 5.6686003517825156e-05, 5.3722134907729924e-05, 5.095133383292705e-05, 4.965242987964302e-05, 7.133147300919518e-05, 6.81934179738164e-05, 6.719360681017861e-05, 6.452809611801058e-05, 8.349500421900302e-05, 7.972883031470701e-05, 7.844359060982242e-05, 7.669772458029911e-05, 9.192626021103933e-05, 8.915743819670752e-05, 8.75290934345685e-05, 8.475895447190851e-05, 0.00010006802040152252, 9.642801887821406e-05, 9.452445374336094e-05, 9.257125202566385e-05, 0.00010157848009839654, 0.00010380306048318744, 9.915343252941966e-05, 9.759078966453671e-05, 0.00010898542677750811, 0.00010359243606217206, 0.00010322066373191774, 0.00010101655061589554, 0.00010964092507492751, 0.0001075521795428358, 0.00010417013254482299, 0.00010416879376862198 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.289650744089158e-05, "first_to_last_ratio": 0.5036607689093815, "imbalance_abs_log_ratio": 0.6858523150899144, "last_quartile_mean": 0.00010502407712920103, "mean": 8.374395429200376e-05, "normalized": [ 0.5462883602941595, 0.5343271256807102, 0.5249643991287053, 0.5130648517134513, 0.7580131602578075, 0.7348274620741431, 0.7341801809099809, 0.7074999463902408, 0.9290893112957194, 0.896130362600229, 0.8896360907643062, 0.8667732512123232, 1.0501229826066591, 1.0276121076179003, 1.01304723882372, 0.9917393510831477, 1.161342307568102, 1.1291245609681106, 1.1079392656735292, 1.0938109858524918, 1.2035226235600145, 1.226263478849651, 1.1724755435916903, 1.1553301515952579, 1.2932755854998876, 1.227360989677261, 1.2282904680523223, 1.203004103561663, 1.301298744133899, 1.2818290480161694, 1.2464280682225441, 1.251387892724203 ], "peak_layer": 29, "peak_normalized": 1.301298744133899, "population_cv": 0.24714813814810152, "rest_mean": 8.048455465878187e-05, "spike_contrast": 1.2591821218500583, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010134471231140196, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 4.574834747472778e-05, 4.474666638998315e-05, 4.396259464556351e-05, 4.296607949072495e-05, 6.347901944536716e-05, 6.153735739644617e-05, 6.14831515122205e-05, 5.924884317209944e-05, 7.780561281833798e-05, 7.504550012527034e-05, 7.450164412148297e-05, 7.258701953105628e-05, 8.794145105639473e-05, 8.60563013702631e-05, 8.483658166369423e-05, 8.305217488668859e-05, 9.725539712235332e-05, 9.455735562369227e-05, 9.278321522288024e-05, 9.160005720332265e-05, 0.00010078774357680231, 0.00010269215272273868, 9.818773833103478e-05, 9.675191540736705e-05, 0.00010830401151906699, 0.00010278406261932105, 0.00010286190081387758, 0.00010074432066176087, 0.00010897590254899114, 0.00010734543320722878, 0.00010438081517349929, 0.00010479617048986256 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010133173600479495, "first_to_last_ratio": 0.7712367006907566, "imbalance_abs_log_ratio": 0.25975994774816064, "last_quartile_mean": 0.00013138863323547412, "mean": 0.00011757463107642252, "normalized": [ 0.9240254436843522, 0.8516610763218788, 0.7400083948384197, 0.7305203183107621, 0.9945100925181989, 0.9162256174012862, 0.9029060711458456, 0.8349457218511338, 1.057384774375838, 0.9561549232760007, 0.9255087968738563, 0.9289705112696707, 1.0651337322266232, 1.006401362110571, 0.966928758219108, 0.9306903222204219, 1.1004660046561014, 1.0434523934672617, 1.009934985409439, 0.9657192405683421, 1.061624429236619, 1.0862247001295862, 1.0383026763618062, 1.022368851612807, 1.1429833503196458, 1.0926891991234107, 1.0888718388877934, 1.0734199646869733, 1.177032649101865, 1.140664442905499, 1.1108444025161495, 1.1134249543727335 ], "peak_layer": 29, "peak_normalized": 1.177032649101865, "population_cv": 0.10936728326378568, "rest_mean": 0.00011604396611262389, "spike_contrast": 1.0844184846181808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012584022188093512, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.00010864195064641535, 0.00010013373685069382, 8.700621401658282e-05, 8.58906569192186e-05, 0.00011692915722960606, 0.00010772488894872367, 0.0001061588482116349, 9.816843521548435e-05, 0.0001243216247530654, 0.00011241956235608086, 0.00010881635535042733, 0.00010922336514340714, 0.00012523270561359823, 0.00011832726886495948, 0.00011368629202479497, 0.00010942557128146291, 0.0001293868845095858, 0.00012268353020772338, 0.00011874273332068697, 0.00011354408343322575, 0.00012482010060921311, 0.00012771246838383377, 0.0001220780541189015, 0.00012020464055240154, 0.00013438584574032575, 0.0001284725294681266, 0.00012802370474673808, 0.0001262069563381374, 0.00013838917948305607, 0.00013411320105660707, 0.00013060712080914527, 0.00013091052824165672 ] } } } } }, "learned": { "bits_per_byte": 8.06433666787865, "forward": { "activation_sha256": { "attention_branch_output": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", "mlp_branch_output": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", "post_attention_state": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", "post_mlp_state": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", "pre_attention_input": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", "pre_mlp_input": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248" }, "logits_sha256": "30ae47372d19e5c17da2b2a4846b2d27ef6e44fdc06045752d040a56bc6e9398", "loss_nats": 5.5897722244262695, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.5897722244262695, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0021227982360869646, 0.00224129855632782, 0.0024903982412070036, 0.0023722455371171236, 0.0023577234242111444, 0.0023993616923689842, 0.002688874490559101, 0.0029664887115359306, 0.0029547461308538914, 0.003052416956052184, 0.0028214550111442804, 0.0032544767018407583, 0.00348988501355052, 0.003914494998753071, 0.0035071265883743763, 0.004211232997477055, 0.004072706215083599, 0.004050548188388348, 0.003929397091269493, 0.004410331137478352, 0.004824656527489424, 0.00433384021744132, 0.004475373309105635, 0.004755842965096235, 0.004843301139771938, 0.0050001065246760845, 0.005346097983419895, 0.004958339501172304, 0.0054801348596811295, 0.005455824080854654, 0.005286100786179304, 0.005402065813541412 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015727665777376387, "first_to_last_ratio": 3.444367030996094, "imbalance_abs_log_ratio": 1.2367401520242614, "last_quartile_mean": 4.566199140754179e-05, "mean": 8.846047569477378e-05, "normalized": [ 2.4938583213844665, 2.2321521039252294, 1.9412946714691228, 1.7875630514690435, 1.605255150569201, 1.4809330135294976, 1.3782219007164203, 1.3041730210393685, 1.1983684502302745, 1.1165155187552953, 1.0705163494358503, 1.028915549335682, 0.969265519878856, 0.9296915475356099, 0.8760186941641476, 0.840544602365157, 0.8215144482528359, 0.7958412144142111, 0.7578169321029499, 0.7149281644739428, 0.6789722877716906, 0.648159768212111, 0.6127699836305072, 0.5872278577962867, 0.5712871725092085, 0.5525140937217092, 0.5351005407841478, 0.5196042162209058, 0.49974197043571306, 0.49405312503381205, 0.48454882285128825, 0.4726319359854579 ], "peak_layer": 1, "peak_normalized": 2.4938583213844665, "population_cv": 0.5254866190950809, "rest_mean": 9.469065830840177e-05, "spike_contrast": 0.5789112734082525, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.48174895811826e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00022060789342503995, 0.0001974572369363159, 0.00017172785010188818, 0.00015812867786735296, 0.00014200163423083723, 0.0001310040388489142, 0.00012191816495032981, 0.00011536776582943276, 0.00010600824316497892, 9.876749390969053e-05, 9.469838551012799e-05, 9.101835894398391e-05, 8.574168896302581e-05, 8.224095654441044e-05, 7.749303040327504e-05, 7.435497536789626e-05, 7.267155888257548e-05, 7.040049240458757e-05, 6.703684630338103e-05, 6.324288551695645e-05, 6.0062211559852585e-05, 5.733652142225765e-05, 5.42059242434334e-05, 5.1946455641882494e-05, 5.053633503848687e-05, 4.887565955868922e-05, 4.733524838229641e-05, 4.596443613991141e-05, 4.420741242938675e-05, 4.370417445898056e-05, 4.286341936676763e-05, 4.180924588581547e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00022213962802197784, "first_to_last_ratio": 1.7678973829107683, "imbalance_abs_log_ratio": 0.5697909211909248, "last_quartile_mean": 0.00012565187898871955, "mean": 0.00016188367408176418, "normalized": [ 1.7325269150218519, 1.5951124185334453, 1.452057969341735, 1.3730142236731557, 1.2857984704138503, 1.2257558334519054, 1.176224442377172, 1.1372505911497117, 1.088955563427776, 1.050933480062714, 1.0259126283978994, 1.0049209692997896, 0.9780904283068869, 0.9585002626676595, 0.9364797090707767, 0.9194681619525875, 0.9067005574410794, 0.8936181540011504, 0.876961137714969, 0.8605080845266135, 0.8461083257765747, 0.8336465310750286, 0.8211746685608088, 0.8107906186997867, 0.8025238658005067, 0.7926026656489187, 0.7851375623337778, 0.7775608141673811, 0.7689788128837574, 0.7655650153667295, 0.7607140375905382, 0.7564070812634623 ], "peak_layer": 1, "peak_normalized": 1.7325269150218519, "population_cv": 0.2510633514944888, "rest_mean": 0.00016719439386010722, "spike_contrast": 0.7967120440064878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001332057872787118, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002804678224492818, 0.0002582226588856429, 0.00023506447905674577, 0.00022226858709473163, 0.00020814978051930666, 0.00019842985784634948, 0.00019041153427679092, 0.00018410230404697359, 0.00017628412751946598, 0.0001701289729680866, 0.0001660785055719316, 0.0001626802986720577, 0.0001583368721185252, 0.00015516554412897676, 0.00015160077600739896, 0.00014884688425809145, 0.0001467800175305456, 0.00014466218999587, 0.00014196569100022316, 0.0001393022103002295, 0.00013697112444788218, 0.00013495376333594322, 0.0001329347724094987, 0.0001312537642661482, 0.00012991551193408668, 0.00012830943160224706, 0.0001271009532501921, 0.00012587440141942352, 0.0001244851155206561, 0.00012393247743602842, 0.0001231471833307296, 0.00012244995741639286 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19699564762413502, "first_to_last_ratio": 1.7678974461601602, "imbalance_abs_log_ratio": 0.5697909569675436, "last_quartile_mean": 0.11142934113740921, "mean": 0.14356006286107004, "normalized": [ 1.73252692376823, 1.5951124529267644, 1.4520579427898739, 1.3730142509369438, 1.2857984860257277, 1.2257558423238033, 1.1762244498747965, 1.1372506023188744, 1.0889555406950477, 1.0509335152923371, 1.0259126331612176, 1.004920958483027, 0.9780904724894115, 0.9585002702989701, 0.9364797557971557, 0.9194681924019411, 0.9067005932791752, 0.893618176643793, 0.876961081367795, 0.8605080517791706, 0.8461083423470157, 0.8336465303631113, 0.8211746500314575, 0.8107906024915998, 0.8025238662049899, 0.792602599370704, 0.7851375422989607, 0.7775607992253795, 0.7689787783378558, 0.7655650375651303, 0.7607140142018359, 0.7564070449079043 ], "peak_layer": 1, "peak_normalized": 1.73252692376823, "population_cv": 0.25106336091926634, "rest_mean": 0.1482696624265777, "spike_contrast": 0.7967120399011176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1181282252073288, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2487216740846634, 0.2289944440126419, 0.20845752954483032, 0.19711001217365265, 0.18458931148052216, 0.17596958577632904, 0.1688588559627533, 0.16326376795768738, 0.15633052587509155, 0.15087208151817322, 0.14728008210659027, 0.1442665159702301, 0.14041472971439362, 0.13760235905647278, 0.1344410926103592, 0.1319989114999771, 0.13016599416732788, 0.1282878816127777, 0.12589658796787262, 0.12353459000587463, 0.12146736681461334, 0.11967834830284119, 0.11788788437843323, 0.11639714986085892, 0.11521037667989731, 0.11378607898950577, 0.11271439492702484, 0.11162667721509933, 0.11039464175701141, 0.10990456491708755, 0.10920815169811249, 0.10858984291553497 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.813340178268845e-05, "first_to_last_ratio": 2.0396781258635253, "imbalance_abs_log_ratio": 0.7127920139678758, "last_quartile_mean": 3.3403996894776355e-05, "mean": 4.817690887648496e-05, "normalized": [ 1.6677253734616262, 1.5880838657379246, 1.4689679818917765, 1.414042908307037, 1.3674754530876347, 1.307279124984285, 1.2642920390991146, 1.2360018052494937, 1.16456296339031, 1.101785452839103, 1.0797656603944665, 1.0672809591929815, 1.0128091896347613, 0.9955804650846075, 0.9623876307655963, 0.9380984486976066, 0.9310658554452689, 0.9082443423860135, 0.8811967490595315, 0.8644115118569954, 0.8497936463301483, 0.8309315019236491, 0.7889969150425813, 0.7623309639866356, 0.7584133539756696, 0.7445182980676538, 0.7180454335344151, 0.7000361319857387, 0.6663708133598921, 0.6665317313809671, 0.6558653811637546, 0.6371080486827607 ], "peak_layer": 1, "peak_normalized": 1.6677253734616262, "population_cv": 0.28258812904813146, "rest_mean": 4.997824995640527e-05, "spike_contrast": 0.7693279992487513, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.844966704491526e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.034585334826261e-05, 7.650897168787196e-05, 7.077033660607412e-05, 6.81242163409479e-05, 6.588074029423296e-05, 6.298066728049889e-05, 6.090968236094341e-05, 5.954674634267576e-05, 5.610504376818426e-05, 5.308061736286618e-05, 5.201977182878181e-05, 5.1418297516647726e-05, 4.879401603830047e-05, 4.796398934558965e-05, 4.6364861191250384e-05, 4.519468348007649e-05, 4.4855874875793234e-05, 4.3756404920713976e-05, 4.245333548169583e-05, 4.1644674638519064e-05, 4.094043106306344e-05, 4.0031711250776425e-05, 3.8011432479834184e-05, 3.672674938570708e-05, 3.653801104519516e-05, 3.586859020288102e-05, 3.459320942056365e-05, 3.372557694092393e-05, 3.210368595318869e-05, 3.21114384860266e-05, 3.159756670356728e-05, 3.069389640586451e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018498800636734813, "first_to_last_ratio": 1.4771173407265819, "imbalance_abs_log_ratio": 0.39009244570614415, "last_quartile_mean": 0.0001252358233614359, "mean": 0.00015057798327688943, "normalized": [ 1.36489476841379, 1.32224224178262, 1.270236278236952, 1.2358216684832493, 1.2002806183282966, 1.1699184301039023, 1.1431585124660382, 1.1216044194279113, 1.0931779382416982, 1.0676137521656621, 1.050334546837667, 1.0352083927879365, 1.0149316924423049, 1.001461761419052, 0.9840691969120363, 0.9695910166861202, 0.9584136856662008, 0.9467149797350989, 0.9322036518549593, 0.9180752132052378, 0.9048391519289843, 0.8925982934025656, 0.8798985862981495, 0.8691051109125614, 0.860352873872478, 0.849675270316085, 0.8415818308236442, 0.8332603200076834, 0.823928337260059, 0.8201054366601405, 0.8147645087933348, 0.8099375145275801 ], "peak_layer": 1, "peak_normalized": 1.36489476841379, "population_cv": 0.15663918075507588, "rest_mean": 0.0001538862707093358, "spike_contrast": 0.8624111204329009, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000132713231141679, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00020552310161292553, 0.00019910057017114013, 0.00019126961706206203, 0.0001860875345300883, 0.00018073583487421274, 0.00017616395780351013, 0.0001721345033729449, 0.00016888893151190132, 0.0001646085293032229, 0.00016075912571977824, 0.00015815725782886147, 0.0001558795920573175, 0.00015282636741176248, 0.00015079809236340225, 0.0001481791550759226, 0.00014599905989598483, 0.00014431599993258715, 0.00014255443238653243, 0.00014036934589967132, 0.000138241914100945, 0.00013624885468743742, 0.00013440565089695156, 0.0001324933546129614, 0.0001308680948568508, 0.00012955020065419376, 0.00012794238864444196, 0.0001267236948478967, 0.00012547065853141248, 0.00012406546738930047, 0.00012348982272669673, 0.0001226855965796858, 0.00012195875751785934 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015959660413500387, "first_to_last_ratio": 1.2728751185715401, "imbalance_abs_log_ratio": 0.2412782146622287, "last_quartile_mean": 0.00012538276678242255, "mean": 0.00014144585429676226, "normalized": [ 1.173955566226187, 1.1612997099437044, 1.147257651804958, 1.1326524298721496, 1.1221543736292459, 1.1091111750587395, 1.0952125313023318, 1.0849401932479867, 1.070907908686229, 1.0558579375605581, 1.0430997159190893, 1.0346471150202732, 1.0228091528121042, 1.0150746526225596, 1.0038187840838828, 0.9945868690199192, 0.986355871119351, 0.9777591356798918, 0.9676015085554662, 0.9595938626090281, 0.9511867345631487, 0.9428977124791953, 0.9335239262292595, 0.9222035520824579, 0.9142471402553919, 0.9038764491526137, 0.8952592406413572, 0.8872171299760644, 0.877779969795339, 0.8748896658804006, 0.8709399062198925, 0.8672824279512239 ], "peak_layer": 1, "peak_normalized": 1.173955566226187, "population_cv": 0.09275081554992197, "rest_mean": 0.00014320575994335942, "spike_contrast": 0.9213481626529775, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013194236380513756, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016605114797130227, 0.0001642610295675695, 0.0001622748386580497, 0.00016020899056456983, 0.00015872408403083682, 0.00015687917766626924, 0.0001549132721265778, 0.00015346029249485582, 0.00015147548401728272, 0.00014934672799427062, 0.00014754213043488562, 0.000146346545079723, 0.00014467211440205574, 0.00014357810141518712, 0.00014198600547388196, 0.00014068018936086446, 0.00013951594883110374, 0.00013829997624270618, 0.00013686322199646384, 0.0001357305736746639, 0.00013454142026603222, 0.0001333689724560827, 0.00013204308925196528, 0.00013044186925981194, 0.00012931646779179573, 0.00012784957652911544, 0.00012663070810958743, 0.000125493184896186, 0.0001241583377122879, 0.00012374951620586216, 0.0001231908390764147, 0.00012267350393813103 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031297675377572887, "first_to_last_ratio": 2.208693462790297, "imbalance_abs_log_ratio": 0.7924011473984336, "last_quartile_mean": 0.00014170221402309835, "mean": 0.00020716176595669822, "normalized": [ 1.8096319791310764, 1.7504284704923412, 1.6109023900661548, 1.5434988529701714, 1.4353862656069223, 1.3677596328902204, 1.3131374518280827, 1.2555297512219317, 1.1870915168673883, 1.122656500215855, 1.088514230207685, 1.04408820740568, 0.9901334868561585, 0.9607872809839492, 0.9219812869372472, 0.880015778430751, 0.8614610505244651, 0.8397919049431777, 0.8116424497290868, 0.7845312039842096, 0.7619415782348183, 0.7436794877201931, 0.7280047725498856, 0.715266757195506, 0.7065993237444729, 0.696468071137219, 0.6907565141722826, 0.6863265628748565, 0.6782234011874358, 0.6754056248242399, 0.6708832321424102, 0.6674749829241257 ], "peak_layer": 1, "peak_normalized": 1.8096319791310764, "population_cv": 0.33826602846938675, "rest_mean": 0.00021747771663487784, "spike_contrast": 0.6964190843920172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001514556322945282, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00037488655652850866, 0.0003626218531280756, 0.0003337173839099705, 0.0003197539481334388, 0.0002973571536131203, 0.00028334750095382333, 0.00027203187346458435, 0.0002600977604743093, 0.0002459199749864638, 0.00023257150314748287, 0.00022549853019881994, 0.0002162951568607241, 0.00020511780166998506, 0.00019903838983736932, 0.00019099927158094943, 0.00018230562272947282, 0.00017846179252956063, 0.00017397277406416833, 0.0001681412832112983, 0.0001625248696655035, 0.0001578451629029587, 0.00015406195598188788, 0.00015081475430633873, 0.0001481759245507419, 0.00014638036373071373, 0.0001442815555492416, 0.0001430983393220231, 0.00014218062278814614, 0.00014050195750314742, 0.00013991822197567672, 0.00013898135512135923, 0.00013827529619447887 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027448106557130814, 0.0027243648655712605, 0.0027103275060653687, 0.0027234479784965515, 0.0027595730498433113, 0.0027272943407297134, 0.002728298306465149, 0.0027007071767002344, 0.002684502862393856, 0.002683063969016075, 0.0027139110025018454, 0.002666700165718794, 0.002679243916645646, 0.0026584581937640905, 0.002684473292902112, 0.0026407993864268064, 0.002663099905475974, 0.0026283918414264917, 0.002597011858597398, 0.0026434636674821377, 0.0025708195753395557, 0.002691218862310052, 0.0026598076801747084, 0.0026596153620630503, 0.0026765079237520695, 0.002582729561254382, 0.002614464145153761, 0.0025760054122656584, 0.002634295728057623, 0.00259007909335196, 0.00257018837146461, 0.0025643527042120695 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.000156725715896755, "first_to_last_ratio": 3.4451295459637388, "imbalance_abs_log_ratio": 1.236961507811942, "last_quartile_mean": 4.549196592051885e-05, "mean": 8.812961027615529e-05, "normalized": [ 2.4952457548653, 2.231881013356295, 1.939989240144028, 1.7866526882708733, 1.6079164676041713, 1.4797016884434102, 1.3811975174028512, 1.3042540698565042, 1.1954344722384815, 1.1168849900912188, 1.0692220969851183, 1.0304138184362992, 0.9676518223611118, 0.9260610329319834, 0.8738547948342114, 0.8395441231817105, 0.8221010666653766, 0.7971164256793037, 0.7591159941756684, 0.7162115141810864, 0.6791638220644127, 0.6489084022271779, 0.6150954832405342, 0.5868306199881643, 0.5699702288291514, 0.552179306852942, 0.5335984532948914, 0.5198944860736878, 0.5024723171712969, 0.4942325662695092, 0.4850276094601851, 0.47217611282304317 ], "peak_layer": 1, "peak_normalized": 2.4952457548653, "population_cv": 0.5254777238950282, "rest_mean": 9.433142622583546e-05, "spike_contrast": 0.5792322488273548, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.463980414788239e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021990503591950983, 0.00019669480388984084, 0.00017097049567382783, 0.00015745700511615723, 0.00014170505164656788, 0.0001304055331274867, 0.00012172439892310649, 0.00011494340287754312, 0.00010535317414905876, 9.843063890002668e-05, 9.423012670595199e-05, 9.080996824195608e-05, 8.527877798769623e-05, 8.16133979242295e-05, 7.701248250668868e-05, 7.398869638564065e-05, 7.24514466128312e-05, 7.024955993983895e-05, 6.690059672109783e-05, 6.311944162007421e-05, 5.985444295220077e-05, 5.718804459320381e-05, 5.420812522061169e-05, 5.1717153837671503e-05, 5.023125413572416e-05, 4.866334711550735e-05, 4.7025823732838035e-05, 4.581809844239615e-05, 4.428268948686309e-05, 4.3556523451115936e-05, 4.274529419490136e-05, 4.161269680480473e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002203928706876468, "first_to_last_ratio": 1.7624395804596433, "imbalance_abs_log_ratio": 0.5666989745264205, "last_quartile_mean": 0.00012504988717410015, "mean": 0.00016078790008577926, "normalized": [ 1.7301295632680096, 1.5926765618359837, 1.4498368649995454, 1.3715838442653223, 1.2853282969253275, 1.2242662006350955, 1.1759759999659367, 1.1358472505674493, 1.0864560712260232, 1.0500344477950443, 1.0256300088344572, 1.0049332388641516, 0.9775387355078093, 0.9578140668049778, 0.9358065303644342, 0.9184543281568608, 0.9072826301440372, 0.8934865935730769, 0.8779252804708734, 0.8613603707803614, 0.8467225584074481, 0.8352602527072714, 0.8224752340358507, 0.8113194646910352, 0.80159772798593, 0.7940537841119606, 0.785769247679408, 0.7781743501685717, 0.7724121545166266, 0.7676777199663887, 0.7637141967462265, 0.7584564239985051 ], "peak_layer": 1, "peak_normalized": 1.7301295632680096, "population_cv": 0.2498913679417364, "rest_mean": 0.0001660440253477578, "spike_contrast": 0.7974079368034485, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013240482367109506, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027818389935418963, 0.00025608311989344656, 0.00023311622499022633, 0.00022053408611100167, 0.00020666523778345436, 0.0001968471915461123, 0.00018908271158579737, 0.0001826304942369461, 0.0001746889902278781, 0.00016883283387869596, 0.00016490889538545161, 0.00016158110520336777, 0.00015717640053480864, 0.00015400491247419268, 0.0001504663669038564, 0.00014767634274903685, 0.00014588006888516247, 0.00014366183313541114, 0.00014115976227913052, 0.00013849632523488253, 0.00013614274212159216, 0.0001342997420579195, 0.00013224406575318426, 0.00013045035302639008, 0.00012888721539638937, 0.00012767424050252885, 0.0001263421872863546, 0.0001251210196642205, 0.00012419452832546085, 0.00012343328853603452, 0.00012279600196052343, 0.00012195061572128907 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19544659927487373, "first_to_last_ratio": 1.7624395617194093, "imbalance_abs_log_ratio": 0.5666989638932991, "last_quartile_mean": 0.11089549027383327, "mean": 0.14258831785991788, "normalized": [ 1.7301295593799786, 1.5926765086027146, 1.4498368632986096, 1.3715838909156086, 1.2853282557768668, 1.224266111161914, 1.1759760185820738, 1.1358472270183295, 1.0864560720464596, 1.050034483149275, 1.025630003735948, 1.0049332495130952, 0.9775386792530747, 0.9578141330250067, 0.9358065753109587, 0.9184542876534186, 0.9072827263675407, 0.8934866319653527, 0.8779253472761699, 0.8613603464001749, 0.8467225704769723, 0.8352602246328348, 0.8224752140046494, 0.8113194329409806, 0.8015977187598701, 0.7940537797484172, 0.7857692153409586, 0.7781743825002613, 0.7724121455314925, 0.7676776608495898, 0.7637142124554999, 0.758456472325904 ], "peak_layer": 1, "peak_normalized": 1.7301295593799786, "population_cv": 0.24989136150327743, "rest_mean": 0.14724950271624107, "spike_contrast": 0.7974079196861148, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11741791963577271, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24669626355171204, 0.2270970642566681, 0.20672979950904846, 0.19557183980941772, 0.18327279388904572, 0.17456604540348053, 0.16768044233322144, 0.16195854544639587, 0.1549159437417984, 0.1497226506471634, 0.14624285697937012, 0.14329174160957336, 0.13938559591770172, 0.13657310605049133, 0.13343508541584015, 0.1309608519077301, 0.1293679177761078, 0.12740075588226318, 0.1251818984746933, 0.1228199228644371, 0.12073274701833725, 0.11909835040569305, 0.11727535724639893, 0.11568467319011688, 0.11429847031831741, 0.11322279274463654, 0.11204151064157486, 0.11095857620239258, 0.11013694852590561, 0.10946186631917953, 0.10889672487974167, 0.108147032558918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.812671563238837e-05, "first_to_last_ratio": 2.0429267595554257, "imbalance_abs_log_ratio": 0.7143834657339574, "last_quartile_mean": 3.334760549478233e-05, "mean": 4.8136708642232406e-05, "normalized": [ 1.6706878479333491, 1.5890573277727331, 1.4709453763829752, 1.4135957551070697, 1.368779351628352, 1.3072249852707734, 1.2659085053079855, 1.2360067186760548, 1.1647282180273046, 1.1006681301372137, 1.0750864931045292, 1.0685694254166505, 1.0112110373273602, 0.9945647481091522, 0.9602230239537092, 0.9352227929155844, 0.93139607887314, 0.9092220870879134, 0.8814974932144994, 0.8653605860843365, 0.8506124631158822, 0.8320765477333729, 0.7932580049774197, 0.7619473309737773, 0.7566723544753674, 0.7419402536141336, 0.7141691805145849, 0.7010673290630277, 0.6716446941281601, 0.6659218543873533, 0.6557479674543484, 0.6349860372318865 ], "peak_layer": 1, "peak_normalized": 1.6706878479333491, "population_cv": 0.2830124551063321, "rest_mean": 4.992923641111702e-05, "spike_contrast": 0.7702312603701833, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.845705869025551e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.04214141680859e-05, 7.649198960280046e-05, 7.080646901158616e-05, 6.804584700148553e-05, 6.588853284483775e-05, 6.292550824582577e-05, 6.093666888773441e-05, 5.949729529675096e-05, 5.6066182878566906e-05, 5.298254109220579e-05, 5.1751125283772126e-05, 5.143741509527899e-05, 4.867637107963674e-05, 4.7875073505565524e-05, 4.622197593562305e-05, 4.501854709815234e-05, 4.483434167923406e-05, 4.376695869723335e-05, 4.24323879997246e-05, 4.165561040281318e-05, 4.094568430446088e-05, 4.0053426346275955e-05, 3.8184829463716596e-05, 3.667763667181134e-05, 3.642371666501276e-05, 3.571456181816757e-05, 3.4377753763692454e-05, 3.3747073757695034e-05, 3.2330764952348545e-05, 3.205528628313914e-05, 3.156554885208607e-05, 3.056613786611706e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018337754227104597, "first_to_last_ratio": 1.4712162101549686, "imbalance_abs_log_ratio": 0.38608941256902984, "last_quartile_mean": 0.00012464350311347516, "mean": 0.00014952537844692415, "normalized": [ 1.3620834570997993, 1.3194965005671844, 1.2677944868152182, 1.233561928872588, 1.1984447257225705, 1.1684695593357564, 1.141528073961558, 1.1198008373135866, 1.0910012077210556, 1.066096352737946, 1.0498630645220353, 1.0350823851472097, 1.0145919973375974, 1.0013407128400098, 0.9836056688270479, 0.9685428567287663, 0.9591027483551056, 0.9465833155582388, 0.9334001555540693, 0.9191584382428863, 0.9057431683575706, 0.8945627711391904, 0.8815040843165663, 0.8698871064160805, 0.8596264871795505, 0.8514885298372257, 0.8425068993042694, 0.8341757603627517, 0.8278236379550317, 0.8225948884437775, 0.8181940461689637, 0.812344147258793 ], "peak_layer": 1, "peak_normalized": 1.3620834570997993, "population_cv": 0.15505530299928885, "rest_mean": 0.00015278545470209792, "spike_contrast": 0.8634393039979241, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001319209666689858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00020366604439914227, 0.0001972982136067003, 0.0001895674504339695, 0.00018444881425239146, 0.00017919790116138756, 0.00017471585306338966, 0.0001706874172668904, 0.0001674386439844966, 0.00016313236847054213, 0.00015940846060402691, 0.00015698117204010487, 0.00015477108536288142, 0.00015170725237112492, 0.00014972584904171526, 0.0001470740098739043, 0.0001448217371944338, 0.00014341020141728222, 0.00014153822849038988, 0.00013956701150164008, 0.00013743751333095133, 0.00013543159002438188, 0.00013375983689911664, 0.00013180723180994391, 0.00013007019879296422, 0.00012853597581852227, 0.00012731914466712624, 0.0001259761629626155, 0.00012473044625949115, 0.00012378064275253564, 0.00012299881200306118, 0.00012234077439643443, 0.00012146606604801491 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015816818631719798, "first_to_last_ratio": 1.2671705792413261, "imbalance_abs_log_ratio": 0.23678652466939742, "last_quartile_mean": 0.00012481996418500785, "mean": 0.0001404537983944465, "normalized": [ 1.172032188999453, 1.1600674087975105, 1.1434054243316032, 1.1305758413418425, 1.1202630655069494, 1.1065457827105485, 1.0931998253234612, 1.0828906757132684, 1.0683311763286112, 1.0526488941703125, 1.0427563459749776, 1.0343706495522589, 1.0219941374371992, 1.0154952175292713, 1.0038357654470964, 0.9937321709742023, 0.9862865992066487, 0.9768881500283385, 0.968261257859838, 0.9607910277643221, 0.9520546236098718, 0.9457805296333641, 0.9349892993508679, 0.9232794945485685, 0.9140411172891727, 0.9051057888083044, 0.8959794100889387, 0.888100763485746, 0.8822517666505367, 0.8784085901067091, 0.87543446421392, 0.8702025472162866 ], "peak_layer": 1, "peak_normalized": 1.172032188999453, "population_cv": 0.09096684083762872, "rest_mean": 0.00014216970130628734, "spike_contrast": 0.9227558436510851, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013118792267050595, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016461637278553098, 0.0001629358739592135, 0.00016059563495218754, 0.00015879367128945887, 0.0001573452027514577, 0.00015541855827905238, 0.00015354406787082553, 0.00015209610864985734, 0.00015005117165856063, 0.0001478485355619341, 0.0001464590895920992, 0.00014528128667734563, 0.00014354295853991061, 0.00014263016055338085, 0.00014099254622124135, 0.00013957345800008625, 0.0001385276991641149, 0.00013720765127800405, 0.00013599597150459886, 0.00013494674931280315, 0.00013371968816500157, 0.00013283846783451736, 0.00013132279855199158, 0.0001296781119890511, 0.0001283805468119681, 0.00012712554598692805, 0.0001258437114302069, 0.00012473712558858097, 0.00012391561176627874, 0.0001233758230227977, 0.0001229580957442522, 0.00012222325312905014 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0003110146244580392, "first_to_last_ratio": 2.2098355110548322, "imbalance_abs_log_ratio": 0.7929180833727034, "last_quartile_mean": 0.00014074107457417995, "mean": 0.00020577548139044666, "normalized": [ 1.8085529821987474, 1.7406694730308092, 1.619926374791616, 1.5420513399817848, 1.4477144166533082, 1.3679714513872108, 1.309931951281633, 1.254598241859695, 1.1926238819658395, 1.119002359728262, 1.082854716042971, 1.0468411526236405, 0.9850613364068194, 0.9618428208148979, 0.9182052809224994, 0.877960480924469, 0.8608981306240351, 0.8408442911829569, 0.8148282643879068, 0.7879356949258838, 0.7604349554811285, 0.7438321978222967, 0.727404890131739, 0.7163770004088065, 0.7034258791916428, 0.6971552930824737, 0.6907054382632783, 0.684504508832364, 0.6799757639842041, 0.6759922505973077, 0.6720742215597147, 0.6678029589100581 ], "peak_layer": 1, "peak_normalized": 1.8085529821987474, "population_cv": 0.3383164219384334, "rest_mean": 0.00021605301460820355, "spike_contrast": 0.695555219569954, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001502768020145595, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00037215586053207517, 0.0003581870987545699, 0.00033334112958982587, 0.0003173163568135351, 0.00029790413100272417, 0.0002814949839375913, 0.00026955187786370516, 0.0002581655571702868, 0.00024541275342926383, 0.00023026324925012887, 0.00022282495046965778, 0.00021541424212045968, 0.00020270147069822997, 0.00019792366947513074, 0.00018894413369707763, 0.00018066274060402066, 0.00017715172725729644, 0.00017302513879258186, 0.00016767167835496366, 0.00016213784692808986, 0.0001564788690302521, 0.00015306242858059704, 0.00014968209143262357, 0.00014741282211616635, 0.00014474779891315848, 0.00014345746603794396, 0.00014213024405762553, 0.00014085424481891096, 0.00013992234016768634, 0.00013910263078287244, 0.00013829639647156, 0.00013741747534368187 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0021227982360869646, 0.00413061399012804, 0.005416445434093475, 0.006388718727976084, 0.0023577234242111444, 0.004388594534248114, 0.00574506726115942, 0.007066869642585516, 0.0029547461308538914, 0.004991721361875534, 0.006190058775246143, 0.0074937604367733, 0.00348988501355052, 0.006057556718587875, 0.0074848332442343235, 0.00897275935858488, 0.004072706215083599, 0.0063449000008404255, 0.007735534105449915, 0.00946879480034113, 0.004824656527489424, 0.007074959110468626, 0.009095527231693268, 0.010346688330173492, 0.004843301139771938, 0.007334613241255283, 0.009146629832684994, 0.010587968863546848, 0.0054801348596811295, 0.0077637480571866035, 0.009536821395158768, 0.011367839761078358 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015727686513855588, "first_to_last_ratio": 3.444355141493678, "imbalance_abs_log_ratio": 1.2367367001496994, "last_quartile_mean": 4.5662209231522866e-05, "mean": 8.846079299473786e-05, "normalized": [ 2.4938513501811554, 2.232208910931446, 1.9412909982664082, 1.7875424925486438, 1.6051944492471113, 1.4809426711898561, 1.3782256757419, 1.304162421055249, 1.1983748443875557, 1.116540877408076, 1.0705280549740306, 1.028906183424365, 0.9692497056236429, 0.9297086109864628, 0.876031508599092, 0.840542245421429, 0.8215073067800788, 0.7958492991481336, 0.7578128978794811, 0.7149268338582802, 0.6789684541053794, 0.6481555926886391, 0.6127683614407704, 0.5872234895762031, 0.5712935129257742, 0.5525148673059793, 0.5351006365732414, 0.5196040797164857, 0.49974182292452146, 0.4940518464188538, 0.4845470025727643, 0.4726329960989905 ], "peak_layer": 1, "peak_normalized": 2.4938513501811554, "population_cv": 0.525487396006855, "rest_mean": 9.469102304946218e-05, "spike_contrast": 0.5789096889426617, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.481755069922656e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002206080680480227, 0.00019746297039091587, 0.00017172814114019275, 0.00015812742640264332, 0.00014199677389115095, 0.0001310053630732, 0.00012191893620183691, 0.00011536724196048453, 0.00010600918903946877, 9.877009142655879e-05, 9.469976066611707e-05, 9.101785690290853e-05, 8.574059756938368e-05, 8.224276098189875e-05, 7.74944419390522e-05, 7.435503357555717e-05, 7.267118780873716e-05, 7.040146010695025e-05, 6.70367298880592e-05, 6.324299465632066e-05, 6.0062087868573144e-05, 5.733635771321133e-05, 5.42059751751367e-05, 5.1946255553048104e-05, 5.0537077186163515e-05, 4.88759033032693e-05, 4.733542664325796e-05, 4.596458893502131e-05, 4.4207557948539034e-05, 4.3704218114726245e-05, 4.2863412090810016e-05, 4.1809489630395547e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002221399099653354, "first_to_last_ratio": 1.7678964788399845, "imbalance_abs_log_ratio": 0.5697904098088594, "last_quartile_mean": 0.00012565210272441618, "mean": 0.00016188397967198398, "normalized": [ 1.7325277794978797, 1.5951376332003655, 1.4520548687114199, 1.3729981481664384, 1.2857750985910605, 1.2257666436669268, 1.1762283345939986, 1.1372455678364286, 1.0889526987749056, 1.0509403055231545, 1.0259172538151122, 1.0049179037159137, 0.978085345877984, 0.9585059142501069, 0.9364886382982085, 0.9194669656082258, 0.9066971379228966, 0.893622399915035, 0.8769613699768213, 0.8605074489369314, 0.8461051105300481, 0.8336447776090111, 0.8211750960224249, 0.8107883690330443, 0.8025267555268564, 0.7926031470462279, 0.7851370690218832, 0.7775583575558719, 0.7689758331264998, 0.7655660871494063, 0.7607136802742225, 0.75640826022469 ], "peak_layer": 1, "peak_normalized": 1.7325277794978797, "population_cv": 0.2510639592948361, "rest_mean": 0.00016719473286583606, "spike_contrast": 0.796711177092359, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013320591242518275, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002804684918373823, 0.0002582272281870246, 0.00023506442084908485, 0.00022226640430744737, 0.00020814638992305845, 0.00019843198242597282, 0.00019041252380702645, 0.00018410183838568628, 0.00017628399655222893, 0.00017013039905577898, 0.00016607956786174327, 0.00016268010949715972, 0.00015833634824957699, 0.00015516675193794072, 0.00015160250768531114, 0.00014884697156958282, 0.00014677974104415625, 0.00014466315042227507, 0.00014196599659044296, 0.000139302370371297, 0.00013697086251340806, 0.00013495373423211277, 0.00013293509255163372, 0.00013125364785082638, 0.00012991622497793287, 0.00012830975174438208, 0.00012710111332125962, 0.000125874241348356, 0.00012448486813809723, 0.00012393288488965482, 0.00012314735795371234, 0.00012245037942193449 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19699589163064957, "first_to_last_ratio": 1.7678964443273588, "imbalance_abs_log_ratio": 0.5697903902870003, "last_quartile_mean": 0.11142954230308533, "mean": 0.1435603315476328, "normalized": [ 1.7325277292733527, 1.595137596567507, 1.4520549137362593, 1.3729980837787108, 1.285775112498371, 1.2257666266551688, 1.1762283724960378, 1.1372455675264566, 1.0889526722354506, 1.0509403711345948, 1.0259172522938773, 1.00491793591038, 0.9780853203912144, 0.9585059497750074, 0.9364885904027979, 0.9194669905174069, 0.9066971317518798, 0.8936224205946474, 0.8769613084029552, 0.8605074273314671, 0.84610509802098, 0.833644762521393, 0.821175085273337, 0.8107884103394608, 0.802526775587875, 0.7926031918850489, 0.7851370589142022, 0.7775583578743038, 0.7689758340628073, 0.7655660958716172, 0.7607136803251955, 0.7564082760502372 ], "peak_layer": 1, "peak_normalized": 1.7325277292733527, "population_cv": 0.2510639513231401, "rest_mean": 0.14826996044980156, "spike_contrast": 0.7967111822081809, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11812833547592164, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24872225522994995, 0.22899848222732544, 0.20845748484134674, 0.19710806012153625, 0.184586301445961, 0.17597146332263947, 0.16885973513126373, 0.16326335072517395, 0.156330406665802, 0.1508733481168747, 0.14728102087974548, 0.14426635205745697, 0.1404142528772354, 0.13760343194007874, 0.13444261252880096, 0.13199898600578308, 0.13016574084758759, 0.12828873097896576, 0.1258968561887741, 0.12353473156690598, 0.12146712839603424, 0.1196783185005188, 0.11788816750049591, 0.11639705300331116, 0.11521100997924805, 0.11378637701272964, 0.11271453648805618, 0.111626535654068, 0.1103944256901741, 0.10990492254495621, 0.10920830816030502, 0.10859022289514542 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.813338586653117e-05, "first_to_last_ratio": 2.039661655563216, "imbalance_abs_log_ratio": 0.7127839389845736, "last_quartile_mean": 3.3404258829250466e-05, "mean": 4.817700425974181e-05, "normalized": [ 1.6677237328958603, 1.5880861584886787, 1.4689677920146058, 1.4140322553861728, 1.3674596064698026, 1.3072824267629832, 1.2642946708581568, 1.2359968662296865, 1.1645552208108763, 1.101791124796405, 1.0797798333773554, 1.0672664620429624, 1.012803710836977, 0.9955853656459409, 0.9623936542224847, 0.9380946280712082, 0.9310631059210642, 0.9082494158597965, 0.8812030087749774, 0.8644086677601183, 0.8497891698936241, 0.8309347651333899, 0.7889921814106587, 0.7623244708429415, 0.7584228017816982, 0.7445258855646851, 0.7180452956289476, 0.7000407870401953, 0.6663749309645193, 0.6665299586713209, 0.6558653663654831, 0.637116679476424 ], "peak_layer": 1, "peak_normalized": 1.6677237328958603, "population_cv": 0.28258618188430573, "rest_mean": 4.9978354245131074e-05, "spike_contrast": 0.7693273401931907, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.844971433863975e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.034593338379636e-05, 7.650923362234607e-05, 7.077046757331118e-05, 6.812383799115196e-05, 6.588010728592053e-05, 6.298095104284585e-05, 6.090992974350229e-05, 5.9546626289375126e-05, 5.610478183371015e-05, 5.3080995712662116e-05, 5.2020557632204145e-05, 5.141770088812336e-05, 4.879384869127534e-05, 4.79643204016611e-05, 4.636524317902513e-05, 4.519458889262751e-05, 4.485583122004755e-05, 4.375673597678542e-05, 4.245372110744938e-05, 4.164462006883696e-05, 4.094029645784758e-05, 4.0031947719398886e-05, 3.8011279684724286e-05, 3.672650927910581e-05, 3.653853855212219e-05, 3.5869026760337874e-05, 3.4593271266203374e-05, 3.3725867979228497e-05, 3.21039478876628e-05, 3.211141665815376e-05, 3.1597628549207e-05, 3.0694372981088236e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018498812278266996, "first_to_last_ratio": 1.477115588497286, "imbalance_abs_log_ratio": 0.3900912594562188, "last_quartile_mean": 0.00012523605073511135, "mean": 0.00015057817927299766, "normalized": [ 1.364889512785021, 1.3222437098497068, 1.2702392636002102, 1.235819093505974, 1.2002761568067795, 1.1699209662016437, 1.143160310272995, 1.1216013166366416, 1.0931712967611817, 1.0676161315047876, 1.0503356923443616, 1.0352076251778792, 1.014929888181958, 1.0014649033450271, 0.9840777733314531, 0.96959004456439, 0.958410698647572, 0.9467186761221033, 0.9322053376846741, 0.9180751779000818, 0.904837007766356, 0.8925970349353171, 0.879899470446981, 0.8691033998232655, 0.8603564893899045, 0.8496764837250937, 0.8415817018035472, 0.8332582690137502, 0.8239257185712614, 0.820107268399535, 0.8147646079598274, 0.8099389729407196 ], "peak_layer": 1, "peak_normalized": 1.364889512785021, "population_cv": 0.1566386619724503, "rest_mean": 0.000153886474436149, "spike_contrast": 0.8624109810706347, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013271338539198042, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002055225777439773, 0.00019910105038434267, 0.000191270315553993, 0.00018608738901093602, 0.0001807353983167559, 0.00017616456898394972, 0.00017213499813806266, 0.00016888868412934244, 0.00016460774349980056, 0.00016075969324447215, 0.0001581576361786574, 0.00015587967936880887, 0.00015282629465218633, 0.00015079876175150275, 0.0001481806393712759, 0.00014599910355173051, 0.00014431573799811304, 0.00014255517453420907, 0.00014036978245712817, 0.00013824208872392774, 0.00013624870916828513, 0.00013440563634503633, 0.0001324936602031812, 0.00013086800754535943, 0.00012955091369803995, 0.00012794273789040744, 0.00012672384036704898, 0.0001254705130122602, 0.0001240652345586568, 0.00012349025928415358, 0.00012268577120266855, 0.00012195913586765528 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015959453048708383, "first_to_last_ratio": 1.2728956423420892, "imbalance_abs_log_ratio": 0.24129433847892473, "last_quartile_mean": 0.00012537911607068963, "mean": 0.00014144417764327955, "normalized": [ 1.1740787416792278, 1.1612764386268224, 1.1471117856731012, 1.1325725430810556, 1.1221203501929577, 1.1091452070987, 1.0952451640185767, 1.0850231158766837, 1.0710066115302646, 1.0558964823952193, 1.0428503514266292, 1.0346153464801762, 1.0229188081717178, 1.0151986196356395, 1.003700332970425, 0.9946300373693288, 0.9863513080075261, 0.9777764871930738, 0.967654233599155, 0.9595434059922948, 0.9513191006674802, 0.9429489101168639, 0.9334504238882736, 0.9221966853060849, 0.914156536930625, 0.9038869577772225, 0.8952163547729103, 0.8871546013922188, 0.8776639341179028, 0.8750860454698683, 0.8708633986378481, 0.8673416799041263 ], "peak_layer": 1, "peak_normalized": 1.1740787416792278, "population_cv": 0.09275162400716915, "rest_mean": 0.00014320399646681767, "spike_contrast": 0.9213510743517984, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001319411559961736, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001660666021052748, 0.00016425579087808728, 0.00016225228318944573, 0.0001601957919774577, 0.00015871739014983177, 0.0001568821317050606, 0.0001549160515423864, 0.00015347020234912634, 0.00015148764941841364, 0.0001493504096288234, 0.00014750511036254466, 0.00014634031686000526, 0.00014468590961769223, 0.00014359393389895558, 0.00014196756819728762, 0.0001406846276950091, 0.00013951364962849766, 0.00013830079114995897, 0.0001368690573144704, 0.00013572182797361165, 0.000134558547870256, 0.00013337463315110654, 0.00013203112757764757, 0.00013043935177847743, 0.00012930211960338056, 0.00012784954742528498, 0.00012662314111366868, 0.00012548285303637385, 0.00012414045340847224, 0.00012377582606859505, 0.00012317855725996196, 0.00012268043064977974 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031298474641516805, "first_to_last_ratio": 2.209030513243648, "imbalance_abs_log_ratio": 0.7925537374904509, "last_quartile_mean": 0.00014168421148497146, "mean": 0.0002071673161481158, "normalized": [ 1.8097158340785349, 1.7497804411308835, 1.611244862997228, 1.5435020349400896, 1.4351553463811126, 1.3679605489917586, 1.3130823229189843, 1.255818245825708, 1.1871164694634437, 1.122716192977336, 1.0885067728511792, 1.0444622322416663, 0.990152969051365, 0.9605806666596922, 0.9222549756846732, 0.8800158737132093, 0.8615285136033911, 0.8400222785845712, 0.8115627552032852, 0.7845983398409414, 0.7619432914958448, 0.7435905859348785, 0.7280762324918081, 0.7153162915808863, 0.7064441934578637, 0.6963930075638303, 0.6905250334602637, 0.686293775915277, 0.6782364888338203, 0.6751963304984743, 0.6708841537879804, 0.6673229378400193 ], "peak_layer": 1, "peak_normalized": 1.8097158340785349, "population_cv": 0.3382701486201724, "rest_mean": 0.00021748447411314205, "spike_contrast": 0.6963929896815652, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015145466313697397, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003749139723367989, 0.00036249731783755124, 0.00033379727392457426, 0.0003197631740476936, 0.0002973172813653946, 0.00028339671553112566, 0.0002720277407206595, 0.0002601644955575466, 0.0002459317329339683, 0.00023259010049514472, 0.0002255030267406255, 0.00021637843747157604, 0.00020512733317445964, 0.00019900091865565628, 0.00019106108811683953, 0.00018231052672490478, 0.00017848054994829, 0.00017402516095899045, 0.00016812927788123488, 0.0001625431323191151, 0.00015784974675625563, 0.00015404766600113362, 0.00015083359903655946, 0.00014819015632383525, 0.00014635214756708592, 0.00014426987036131322, 0.00014305421791505069, 0.00014217763964552432, 0.00014050843310542405, 0.0001398786116624251, 0.00013898526958655566, 0.0001382475020363927 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034565660171210766, 0.004928664769977331, 0.0060537648387253284, 0.006960673723369837, 0.0036145171616226435, 0.0051584807224571705, 0.006370538379997015, 0.007546419743448496, 0.00395671371370554, 0.005665866658091545, 0.006755253300070763, 0.007946331985294819, 0.004383862018585205, 0.006652849726378918, 0.007932198233902454, 0.009347881190478802, 0.004880861379206181, 0.006906712893396616, 0.008132148534059525, 0.009816600009799004, 0.005453306715935469, 0.007622495759278536, 0.009448722936213017, 0.010670131072402, 0.0055701653473079205, 0.007738742511719465, 0.009509840048849583, 0.010865328833460808, 0.005987230688333511, 0.00815181527286768, 0.009874933399260044, 0.011666664853692055 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015672678182454547, "first_to_last_ratio": 3.4451487067191406, "imbalance_abs_log_ratio": 1.236967069490208, "last_quartile_mean": 4.549202230919036e-05, "mean": 8.812991916329338e-05, "normalized": [ 2.495342024881723, 2.231858330120814, 1.9399329049931513, 1.7866550124027707, 1.6079479837593746, 1.479673880937976, 1.3811985381554308, 1.3042766606266647, 1.1954508396552082, 1.1168862767589318, 1.0692251193363627, 1.0303878333212484, 0.9676417435164101, 0.9260651349655372, 0.8738635380560305, 0.8395483633300559, 0.8220975248023003, 0.7970985234813915, 0.759112342831592, 0.7162079306537569, 0.6791607399033079, 0.6489075313744581, 0.615087961024076, 0.5868315353418163, 0.5699679008989185, 0.5521854623420828, 0.5335941062986479, 0.5198929115704046, 0.5024718770039331, 0.49422901772262035, 0.4850259920431499, 0.47217445788985424 ], "peak_layer": 1, "peak_normalized": 2.495342024881723, "population_cv": 0.5254832958744828, "rest_mean": 9.433179891655325e-05, "spike_contrast": 0.5792295824234721, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.4639768495690075e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021991429093759507, 0.0001966934942174703, 0.00017096613009925932, 0.0001574577618157491, 0.00014170832582749426, 0.00013040353951510042, 0.0001217249155160971, 0.00011494579666759819, 0.00010535498586250469, 9.843109728535637e-05, 9.423072333447635e-05, 9.080799645744264e-05, 8.527818863512948e-05, 8.161404548445717e-05, 7.701352296862751e-05, 7.398932939395308e-05, 7.245138840517029e-05, 7.024822843959555e-05, 6.690050940960646e-05, 6.311934703262523e-05, 5.985438110656105e-05, 5.7188168284483254e-05, 5.420765228336677e-05, 5.1717415772145614e-05, 5.02312250318937e-05, 4.866406015935354e-05, 4.702560545410961e-05, 4.581812027026899e-05, 4.428280590218492e-05, 4.3556363380048424e-05, 4.2745301470858976e-05, 4.161269680480473e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00022039370378479362, "first_to_last_ratio": 1.7624457042066763, "imbalance_abs_log_ratio": 0.5667024491058967, "last_quartile_mean": 0.00012504992537287762, "mean": 0.0001607882525149762, "normalized": [ 1.7301666786506156, 1.592657504255016, 1.4498087081325215, 1.3715848200709229, 1.2853506396348091, 1.224257634451794, 1.175979576607571, 1.1358564358860717, 1.0864628307036615, 1.050035766385231, 1.0256357250890054, 1.0049217142965674, 0.9775345112735686, 0.9578189361646354, 0.9358080993289649, 0.9184597363064851, 0.9072850761620224, 0.8934833681037547, 0.8779253472396409, 0.8613572157279928, 0.8467159963027754, 0.8352601414820391, 0.8224708066612338, 0.81131886291965, 0.8015955184587156, 0.7940537632065271, 0.7857694259399116, 0.7781749070905981, 0.7724085609032495, 0.7676732316943716, 0.7637136993200662, 0.75845476155001 ], "peak_layer": 1, "peak_normalized": 1.7301666786506156, "population_cv": 0.2498932502086342, "rest_mean": 0.0001660444721454513, "spike_contrast": 0.7974048446155259, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001324046665104106, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027819047681987286, 0.0002560806169640273, 0.0002331122086616233, 0.00022053472639527172, 0.00020666928321588784, 0.00019684624567162246, 0.0001890837011160329, 0.00018263237143401057, 0.00017469045997131616, 0.0001688334159553051, 0.0001649101759539917, 0.00016157960635609925, 0.00015717606584075838, 0.00015400603297166526, 0.00015046694898046553, 0.00014767753600608557, 0.00014588078192900866, 0.00014366162940859795, 0.00014116008242126554, 0.00013849612150806934, 0.0001361419854220003, 0.00013430001854430884, 0.00013224364374764264, 0.00013045054220128804, 0.00012888714263681322, 0.00012767451698891819, 0.0001263424928765744, 0.00012512138346210122, 0.00012419422273524106, 0.00012343283742666245, 0.0001227961911354214, 0.00012195061572128907 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19544734060764313, "first_to_last_ratio": 1.7624457434404193, "imbalance_abs_log_ratio": 0.5667024713668619, "last_quartile_mean": 0.11089552193880081, "mean": 0.14258862822316587, "normalized": [ 1.7301667593069614, 1.5926575752547714, 1.4498087309457477, 1.3715847721481056, 1.2853506436856033, 1.2242575941276408, 1.1759796246815233, 1.1358564592049036, 1.0864627991256137, 1.0500357507634608, 1.0256357136610779, 1.0049216567312906, 0.9775345659226533, 0.957818945514909, 0.935808091557486, 0.9184597083380731, 0.9072850362319335, 0.8934833286128027, 0.8779253174360568, 0.8613572174793438, 0.8467160247668938, 0.8352601831545204, 0.8224708111632975, 0.8113188687979516, 0.8015955037045681, 0.7940537757091718, 0.7857693860910103, 0.7781748832951502, 0.7724085309387246, 0.7676732205256189, 0.7637136996793319, 0.7584548214438025 ], "peak_layer": 1, "peak_normalized": 1.7301667593069614, "population_cv": 0.24989326289316924, "rest_mean": 0.14724989594132812, "spike_contrast": 0.797404859232464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11741778254508972, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24670210480690002, 0.2270948588848114, 0.20672623813152313, 0.1955723911523819, 0.18327638506889343, 0.17456521093845367, 0.16768132150173187, 0.16196021437644958, 0.15491724014282227, 0.14972315728664398, 0.14624398946762085, 0.14329040050506592, 0.13938531279563904, 0.13657408952713013, 0.13343559205532074, 0.13096190989017487, 0.12936852872371674, 0.12740056216716766, 0.1251821666955948, 0.12281974405050278, 0.12073207646608353, 0.11909860372543335, 0.11727498471736908, 0.11568484455347061, 0.11429840326309204, 0.11322303861379623, 0.11204177886247635, 0.11095888912677765, 0.11013667285442352, 0.1094614714384079, 0.1088968887925148, 0.108147032558918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.812703486502869e-05, "first_to_last_ratio": 2.0429306771418236, "imbalance_abs_log_ratio": 0.7143853833664048, "last_quartile_mean": 3.334769780849456e-05, "mean": 4.813682039639389e-05, "normalized": [ 1.670694398734861, 1.5890602892970849, 1.4709330435001327, 1.4135870318536654, 1.3688033811669116, 1.3072210435152478, 1.2659107055367187, 1.236022743120565, 1.164719165633699, 1.10067041168383, 1.075093368589445, 1.0685491843186352, 1.0112041551575939, 0.9945774787180313, 0.9602353052595208, 0.9352302198346567, 0.9313930096398116, 0.9092161974531191, 0.8814894006709005, 0.8653525310040872, 0.8506115464001769, 0.8320776390213925, 0.7932455071661803, 0.7619500965894176, 0.7566697664554205, 0.7419550822306056, 0.7141621566216388, 0.7010675908642553, 0.6716466869044494, 0.66592144202409, 0.6557448579818498, 0.6349845630520059 ], "peak_layer": 1, "peak_normalized": 1.670694398734861, "population_cv": 0.28301355753944435, "rest_mean": 4.992937398061191e-05, "spike_contrast": 0.7702285844110489, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.845703104161657e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.042191620916128e-05, 7.649230974493548e-05, 7.080603973008692e-05, 6.804558506701142e-05, 6.588984251720831e-05, 6.292546459008008e-05, 6.0936916270293295e-05, 5.9498204791452736e-05, 5.606587728834711e-05, 5.2982773922849447e-05, 5.1751576393144205e-05, 5.143656017025933e-05, 4.867615280090831e-05, 4.7875797463348135e-05, 4.622267442755401e-05, 4.501900912146084e-05, 4.483429802348837e-05, 4.3766776798292994e-05, 4.243209696142003e-05, 4.1655319364508614e-05, 4.094573523616418e-05, 4.005357186542824e-05, 3.8184316508704796e-05, 3.667785495053977e-05, 3.6423676647245884e-05, 3.571535853552632e-05, 3.437749546719715e-05, 3.374716470716521e-05, 3.233093593735248e-05, 3.2055340852821246e-05, 3.156547245453112e-05, 3.056613786611706e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018337804431212135, "first_to_last_ratio": 1.471219679743085, "imbalance_abs_log_ratio": 0.38609177087911456, "last_quartile_mean": 0.00012464355040719965, "mean": 0.0001495256622092711, "normalized": [ 1.3620802882826015, 1.3194938018514704, 1.2677903290886852, 1.23356455122679, 1.198458119977671, 1.1684656874236388, 1.1415318441782234, 1.1198031889553155, 1.0910031274146845, 1.0660983196947673, 1.0498697336705507, 1.0350765279986567, 1.0145895852928877, 1.0013439705330982, 0.9836063325227735, 0.968549777523238, 0.9591067674489386, 0.9465820057822166, 0.9334010118478351, 0.9191551367815048, 0.9057362914957724, 0.8945632145340071, 0.8814992971854174, 0.8698868180749819, 0.8596244665411801, 0.8514889576556698, 0.8425072468470839, 0.8341767076391075, 0.8278204125037605, 0.8225904077486038, 0.8181934666462969, 0.8123426056325708 ], "peak_layer": 1, "peak_normalized": 1.3620802882826015, "population_cv": 0.1550560200772002, "rest_mean": 0.00015278581957789829, "spike_contrast": 0.8634362323882038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013192081241868435, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002036659570876509, 0.00019729818450286984, 0.0001895671884994954, 0.0001844495564000681, 0.00017920024401973933, 0.00017471560568083078, 0.00017068830493371934, 0.0001674393133725971, 0.0001631329650990665, 0.00015940905723255128, 0.0001569824671605602, 0.0001547705032862723, 0.00015170717961154878, 0.00014972662029322237, 0.00014707438822370023, 0.00014482304686680436, 0.00014341107453219593, 0.00014153830124996603, 0.00013956740440335125, 0.00013743728050030768, 0.00013543081877287477, 0.00013376015704125166, 0.0001318067661486566, 0.00013007040251977742, 0.00012853591761086136, 0.00012731945025734603, 0.00012597645400092006, 0.00012473082460928708, 0.00012378039536997676, 0.00012299837544560432, 0.0001223409199155867, 0.00012146606604801491 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015817073654034175, "first_to_last_ratio": 1.2671327323753674, "imbalance_abs_log_ratio": 0.23675665700060805, "last_quartile_mean": 0.00012482570491556544, "mean": 0.00014045925900063594, "normalized": [ 1.1719218726461313, 1.1599555891510773, 1.143372783062998, 1.1305925991754215, 1.1202702784385838, 1.106568447670676, 1.0931409560371117, 1.082952696795682, 1.0683580206124534, 1.05266391584401, 1.0427272032106563, 1.034492263470324, 1.0219716035750168, 1.0154196846947314, 1.0039021031599646, 0.9939448772529704, 0.9863124890252286, 0.9768583563343864, 0.9682383265025308, 0.9606909958387225, 0.9520472410590707, 0.9456933063501335, 0.9351354973759991, 0.9231938712502704, 0.9140830768869257, 0.9051693342733999, 0.8960380266364146, 0.8881305740575848, 0.8822109405764296, 0.8783208779569746, 0.8754534745501711, 0.8701687165279489 ], "peak_layer": 1, "peak_normalized": 1.1719218726461313, "population_cv": 0.09095723041158388, "rest_mean": 0.00014217518737832843, "spike_contrast": 0.9227576779061385, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013119324576109647, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016460727783851326, 0.00016292650252580643, 0.00016059729387052357, 0.0001588021987117827, 0.0001573523331899196, 0.00015542778419330716, 0.00015354176866821945, 0.00015211073332466185, 0.00015006077592261136, 0.00014785639359615743, 0.0001464606903027743, 0.00014530401676893234, 0.00014354537415783852, 0.00014262509648688138, 0.0001410073455190286, 0.00013960876094643027, 0.00013853672135155648, 0.0001372088008793071, 0.00013599803787656128, 0.00013493794540409, 0.00013372385001275688, 0.00013283138105180115, 0.00013134843902662396, 0.0001296711270697415, 0.0001283914316445589, 0.00012713941396214068, 0.00012585683725774288, 0.00012474616232793778, 0.00012391469499561936, 0.00012336829968262464, 0.00012296554632484913, 0.00012222325312905014 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031102553475648165, "first_to_last_ratio": 2.2098501973960105, "imbalance_abs_log_ratio": 0.7929247292485178, "last_quartile_mean": 0.00014074507635086775, "mean": 0.00020578417843353236, "normalized": [ 1.8085758305333837, 1.7408578335592815, 1.6197863489261306, 1.5416881776712252, 1.4477928223214513, 1.3679195769143782, 1.3100342829404608, 1.2546744848740619, 1.192398672058626, 1.1190508147725688, 1.082828610043334, 1.0467691899805913, 0.9849335746234156, 0.9618381641711736, 0.9182125099812896, 0.877817233347749, 0.8611427657722097, 0.8409392227987704, 0.8149468534272446, 0.7879793317746896, 0.7604794717340572, 0.7439065501103351, 0.7274095758071225, 0.7164574630456112, 0.7035124756224637, 0.6971408207181957, 0.690610058298894, 0.684463275349207, 0.6799680284055707, 0.6760380727842366, 0.6720531720343774, 0.6677747355978938 ], "peak_layer": 1, "peak_normalized": 1.8085758305333837, "population_cv": 0.33829873005197686, "rest_mean": 0.0002160599310796156, "spike_contrast": 0.6956177084463702, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015029511414468288, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003721762914210558, 0.00035824099904857576, 0.00033332640305161476, 0.00031725503504276276, 0.00029793285648338497, 0.0002814962062984705, 0.0002695843286346644, 0.00025819215807132423, 0.0002453767810948193, 0.00023028295254334807, 0.00022282899590209126, 0.00021540853776969016, 0.00020268374646548182, 0.00019793107639998198, 0.0001889536069938913, 0.00018064089817926288, 0.00017720955656841397, 0.0001730519870761782, 0.00016770316869951785, 0.00016215367941185832, 0.00015649464330635965, 0.00015308419824577868, 0.00014968938194215298, 0.00014743561041541398, 0.00014477173681370914, 0.00014346055104397237, 0.00014211662346497178, 0.00014085171278566122, 0.00013992666208650917, 0.00013911793939769268, 0.00013829790987074375, 0.00013741747534368187 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.028097106143832207, 0.014136734418570995, 0.014224153943359852, 0.014337832108139992, 0.014408928342163563, 0.009704872034490108, 0.009773381985723972, 0.00987160298973322, 0.00994801800698042, 0.007515520788729191, 0.007602147292345762, 0.007658426649868488, 0.007734050042927265, 0.0062688966281712055, 0.006336068734526634, 0.006399397272616625, 0.006462901830673218, 0.005432929843664169, 0.005504137370735407, 0.005560056772083044, 0.005631812382489443, 0.004873365629464388, 0.004937449004501104, 0.005005750339478254, 0.005022916477173567, 0.0044684382155537605, 0.004500201437622309, 0.004573327023535967, 0.00463611725717783, 0.004166214261204004, 0.0042496914975345135, 0.004283483140170574 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00014323742470878642, "first_to_last_ratio": 1.6208859606473658, "imbalance_abs_log_ratio": 0.48297288904334346, "last_quartile_mean": 8.83698348843609e-05, "mean": 0.00011961164659624046, "normalized": [ 0.879665217639624, 1.4482702231163533, 1.4522235545753037, 1.2017346580032677, 1.034069363337856, 1.36550769437152, 1.149442648944982, 1.0492523703464898, 1.0286339735981567, 1.1048881958426662, 1.2960670227766034, 1.1342833655748923, 0.9483258681651744, 1.1110988010941807, 1.0051223270547462, 0.9186295907129635, 0.9340104771989933, 1.0935042558126027, 0.9904421395235656, 1.070541841916413, 0.9831461473937306, 1.1165574278336847, 0.8906099662074067, 0.8835226812669187, 0.8406995671395937, 0.8651479324264472, 0.7705397043177317, 0.6783637574582257, 0.6658814129635636, 0.7912689330389562, 0.6809428819470401, 0.6176059984003462 ], "peak_layer": 3, "peak_normalized": 1.4522235545753037, "population_cv": 0.21063690333645962, "rest_mean": 0.00012087627045629133, "spike_contrast": 0.9330423359872595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011278267775196582, "top_five_layers": [ 3, 2, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010521820513531566, 0.00017322998610325158, 0.00017370285058859736, 0.00014374146121554077, 0.000123686739243567, 0.0001633306237636134, 0.00013748672790825367, 0.00012550280371215194, 0.00012303660332690924, 0.0001321574964094907, 0.00015502471069339663, 0.00013567350106313825, 0.00011343081860104576, 0.0001329003571299836, 0.00012022433656966314, 0.00010987879795720801, 0.0001117185311159119, 0.00013079584459774196, 0.00011846841516671702, 0.00012804927246179432, 0.00011759572953451425, 0.00013355327246244997, 0.00010652732453309, 0.00010567960271146148, 0.00010055745951831341, 0.00010348176874686033, 9.216552280122414e-05, 8.114020602079108e-05, 7.964717224240303e-05, 9.464497998123989e-05, 8.144869934767485e-05, 7.387287041638047e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00015447182067873655, "first_to_last_ratio": 1.6506746305835311, "imbalance_abs_log_ratio": 0.5011840713694677, "last_quartile_mean": 9.358102306578076e-05, "mean": 0.00012785212516064348, "normalized": [ 0.8854544129111614, 1.4719853585239924, 1.4439160513495162, 1.202741955390318, 1.0425045283644727, 1.3767992039076578, 1.1738833181051227, 1.0683704265103389, 1.0407410271769484, 1.136073438345882, 1.2855554885667506, 1.129314905977533, 0.953735223493217, 1.1222184469907663, 1.0164965849441758, 0.9298350247961144, 0.9320412792829462, 1.0837813144211186, 0.9857736034320764, 1.0482233302985868, 0.9656597421723547, 1.0923472818519362, 0.884471876187184, 0.8724974481708365, 0.826330173068434, 0.8536374096219558, 0.7648976975162065, 0.681485935203229, 0.6604246501739865, 0.7781176964145518, 0.6752223987471391, 0.6154627680834903 ], "peak_layer": 2, "peak_normalized": 1.4719853585239924, "population_cv": 0.21455917947613024, "rest_mean": 0.00012955063343893185, "spike_contrast": 0.9160910858365683, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011868018045788631, "top_five_layers": [ 2, 3, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001132072284235619, 0.00018819645629264414, 0.00018460773571860045, 0.00015377311501652002, 0.00013328641944099218, 0.00017602670413907617, 0.0001500834769103676, 0.00013659342948812991, 0.00013306095206644386, 0.00014524940343108028, 0.00016436100122518837, 0.00014438531070481986, 0.00012193707516416907, 0.0001434780133422464, 0.00012996124860364944, 0.00011888138396898285, 0.00011916345829376951, 0.00013856374425813556, 0.00012603325012605637, 0.00013401758042164147, 0.0001234616502188146, 0.00013965892139822245, 0.00011308160901535302, 0.00011155065294587985, 0.00010564806871116161, 0.00010913935693679377, 9.779379615793005e-05, 8.71294250828214e-05, 8.443669503321871e-05, 9.948400111170486e-05, 8.632861863588914e-05, 7.86882228567265e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.13698715530335903, "first_to_last_ratio": 1.6506746463483835, "imbalance_abs_log_ratio": 0.5011840809200188, "last_quartile_mean": 0.08298858627676964, "mean": 0.11338054295629263, "normalized": [ 0.8854544185536968, 1.4719853260144444, 1.4439159993116888, 1.2027420311824657, 1.042504519270847, 1.3767991933658148, 1.1738833593963587, 1.0683704218985235, 1.0407409783116481, 1.1360735196220915, 1.285555449073044, 1.129314933056767, 0.9537352653592515, 1.1222184500196979, 1.0164965450422543, 0.9298350370138166, 0.9320412867497836, 1.0837813135283418, 0.9857735942006465, 1.0482233283425098, 0.9656597514448471, 1.0923472716531482, 0.8844718771212735, 0.8724974491223123, 0.82633015871154, 0.8536374112441639, 0.7648976624892803, 0.681485927333974, 0.660424633911195, 0.7781176812354046, 0.6752224228414072, 0.6154627835777616 ], "peak_layer": 2, "peak_normalized": 1.4719853260144444, "population_cv": 0.2145591770090877, "rest_mean": 0.11488679713673061, "spike_contrast": 0.9160910827435619, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10524677038192749, "top_five_layers": [ 2, 3, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.10039330273866653, 0.16689449548721313, 0.16371197998523712, 0.1363675445318222, 0.11819972842931747, 0.1561022400856018, 0.13309553265571594, 0.12113241851329803, 0.11799977719783783, 0.1288086324930191, 0.14575697481632233, 0.1280423402786255, 0.10813502222299576, 0.1272377371788025, 0.11525093019008636, 0.10542520135641098, 0.1056753471493721, 0.12287971377372742, 0.11176754534244537, 0.11884813010692596, 0.10948702692985535, 0.12385092675685883, 0.10028190165758133, 0.09892423450946808, 0.09368976205587387, 0.09678587317466736, 0.08672451227903366, 0.07726724445819855, 0.07487930357456207, 0.0882234051823616, 0.07655708491802216, 0.06978150457143784 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.7160250005617854e-05, "first_to_last_ratio": 0.7388534944946782, "imbalance_abs_log_ratio": 0.30265562600345663, "last_quartile_mean": 6.382895981005277e-05, "mean": 5.960399772675373e-05, "normalized": [ 0.46158176249046556, 0.8375260596839308, 0.8768757910510345, 0.7241656982213429, 0.6710453705877814, 1.0089930699998053, 0.8842011269201543, 0.865421443073546, 0.883514351886623, 0.9622394496647411, 1.1256912972870226, 1.0210990974149952, 0.9232791878576234, 1.0980011773561495, 1.0212452171652862, 0.9544893655323485, 1.0133393696232265, 1.159814348379829, 0.9921993085906312, 1.1469178460749072, 1.0774269793999733, 1.3924548133021735, 1.1792402184404436, 1.1521666823514416, 1.1069955376913123, 1.1868921569761495, 1.1034570471301723, 0.9847635589252388, 0.9837749617675684, 1.1772881708824627, 1.0498861530819708, 0.9740133811896496 ], "peak_layer": 22, "peak_normalized": 1.3924548133021735, "population_cv": 0.17243044689052606, "rest_mean": 5.759890990264938e-05, "spike_contrast": 1.22279175241262, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.043147197691724e-05, "top_five_layers": [ 22, 26, 23, 30, 18 ], "top_five_spike_overlap": 2 }, "values": [ 2.751211832219269e-05, 4.991990135749802e-05, 5.226530265645124e-05, 4.316317063057795e-05, 3.9996986743062735e-05, 6.014002065057866e-05, 5.2701921958941966e-05, 5.158257772563957e-05, 5.266098742140457e-05, 5.735331797040999e-05, 6.709570152452216e-05, 6.0861588281113654e-05, 5.503113061422482e-05, 6.544525967910886e-05, 6.087029760237783e-05, 5.689138197340071e-05, 6.0399077483452857e-05, 6.912957178428769e-05, 5.9139045333722606e-05, 6.836088869022205e-05, 6.421895523089916e-05, 8.299587352667004e-05, 7.028743129922077e-05, 6.867374031571671e-05, 6.598135951207951e-05, 7.074351742630824e-05, 6.577045132871717e-05, 5.869584492756985e-05, 5.863692058483139e-05, 7.017108146101236e-05, 6.2577411881648e-05, 5.8055091358255595e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6127787956938846e-05, "first_to_last_ratio": 0.8333328618421979, "imbalance_abs_log_ratio": 0.18232212258347721, "last_quartile_mean": 6.735338365615462e-05, "mean": 6.689512088087213e-05, "normalized": [ 0.49022256384162527, 0.923222562655389, 0.9170289550360846, 0.7742286879065733, 0.7345865645537341, 1.0394905128518102, 0.9423822415780315, 0.8911705562121757, 0.9138181637864031, 1.0330628394205372, 1.1540375572503838, 1.0455389170050469, 0.9297420351318767, 1.1341544713967917, 1.0374336268991433, 0.9867016024211971, 1.0073449670971224, 1.1743962796261593, 1.0459325434797282, 1.1475080782429012, 1.066139536390961, 1.3297286391604237, 1.1245973641634917, 1.1027270030076322, 1.04892482254714, 1.125623033590473, 1.0476398534474196, 0.9361019469456514, 0.930671663614635, 1.0896207315037236, 0.9754527388499041, 0.9007689403858299 ], "peak_layer": 22, "peak_normalized": 1.3297286391604237, "population_cv": 0.14905429467389167, "rest_mean": 6.522988486621116e-05, "spike_contrast": 1.1633838617941636, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.58873953600414e-05, "top_five_layers": [ 22, 18, 11, 20, 14 ], "top_five_spike_overlap": 1 }, "values": [ 3.2793497666716576e-05, 6.17590849287808e-05, 6.134476279839873e-05, 5.179212166694924e-05, 4.9140257033286616e-05, 6.953684351174161e-05, 6.304077396634966e-05, 5.961496208328754e-05, 6.112997652962804e-05, 6.910686352057382e-05, 7.719948189333081e-05, 6.994145223870873e-05, 6.219520582817495e-05, 7.586940046167001e-05, 6.939924787729979e-05, 6.600552296731621e-05, 6.738646334270015e-05, 7.856138108763844e-05, 6.996778392931446e-05, 7.676269160583615e-05, 7.13195331627503e-05, 8.895235805539414e-05, 7.523007661802694e-05, 7.37670561647974e-05, 7.016795279923826e-05, 7.529868889832869e-05, 7.00819946359843e-05, 6.26206528977491e-05, 6.225739343790337e-05, 7.289031054824591e-05, 6.525302887894213e-05, 6.025704715284519e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.866022898866504e-05, "first_to_last_ratio": 0.40574519905788725, "imbalance_abs_log_ratio": 0.9020299049238899, "last_quartile_mean": 4.599001795213553e-05, "mean": 3.414976504245715e-05, "normalized": [ 0.2445526901503676, 0.5596630531980494, 0.5376964344799271, 0.48012169469058175, 0.4798804572179837, 0.6900344095639286, 0.7152325131545565, 0.6642055410913166, 0.7554442803991458, 0.9343372865664744, 0.9240467958883766, 0.8592444663674569, 0.8263415713413451, 1.0773994704320806, 0.959597390691018, 0.9894020700335048, 0.9760686528076964, 1.1936439731862338, 1.099336047654388, 1.1448507248259905, 1.0495120137852647, 1.4542665199347329, 1.3223844598378258, 1.2890134714336634, 1.2402412095084396, 1.4030995672846114, 1.3877316376933018, 1.246635573851661, 1.2962845787221184, 1.4847061188341568, 1.3923191445524272, 1.3227061808213745 ], "peak_layer": 30, "peak_normalized": 1.4847061188341568, "population_cv": 0.32662955778396485, "rest_mean": 3.2435424482510255e-05, "spike_contrast": 1.3382653305362568, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.340720406617038e-05, "top_five_layers": [ 30, 22, 26, 31, 27 ], "top_five_spike_overlap": 1 }, "values": [ 8.351416909135878e-06, 1.9112361769657582e-05, 1.8362206901656464e-05, 1.6396043065469712e-05, 1.6387804862461053e-05, 2.3564512957818806e-05, 2.4425022274954244e-05, 2.2682463168166578e-05, 2.5798244678298943e-05, 3.1907398806652054e-05, 3.1555980967823416e-05, 2.9342996640480123e-05, 2.8219370506121777e-05, 3.679293877212331e-05, 3.277002542745322e-05, 3.378784822416492e-05, 3.333251515869051e-05, 4.0762661228654906e-05, 3.7542067730100825e-05, 3.909638326149434e-05, 3.584058868000284e-05, 4.9662859964882955e-05, 4.515911859925836e-05, 4.4019507186021656e-05, 4.2353945900686085e-05, 4.791552055394277e-05, 4.7390709369210526e-05, 4.257231194060296e-05, 4.4267813791520894e-05, 5.070236511528492e-05, 4.754737165058032e-05, 4.517010529525578e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00021511870181711856, "first_to_last_ratio": 1.067203936185183, "imbalance_abs_log_ratio": 0.06504208450186474, "last_quartile_mean": 0.00020157225299044512, "mean": 0.00023711549874860793, "normalized": [ 0.5199215340336985, 0.9694171998286779, 0.9952311992308057, 0.8139231798698612, 0.8071491568326964, 1.1522101416966866, 1.0127543403881063, 0.9872470097646113, 0.9739878350012732, 1.0629045356723543, 1.2654986331851705, 1.158397891601516, 1.0251221075049282, 1.1837639471542862, 1.126357049236326, 1.0240452993420521, 1.0679809871494306, 1.2152134226291609, 1.0519957926786496, 1.2134701299884076, 1.1240302451392508, 1.3270417853241223, 1.0642149202553777, 1.0573092573282454, 0.9986311292560726, 1.01615500686031, 0.8954142860763721, 0.793725079326075, 0.7548043496765433, 0.8883375219775711, 0.764305251609244, 0.6894397743821165 ], "peak_layer": 22, "peak_normalized": 1.3270417853241223, "population_cv": 0.17546356070810173, "rest_mean": 0.00023209894856403548, "spike_contrast": 1.138328593816985, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002642048697452992, "top_five_layers": [ 22, 11, 18, 20, 14 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012328145385254174, 0.00022986384283285588, 0.0002359847421757877, 0.00019299380073789507, 0.00019138757488690317, 0.00027320688241161406, 0.00024013975053094327, 0.00023409156710840762, 0.00023094761127140373, 0.0002520311390981078, 0.00030006933957338333, 0.0002746740938164294, 0.00024307233979925513, 0.00028068877873010933, 0.0002670767134986818, 0.0002428170118946582, 0.00025323484442196786, 0.0002881459367927164, 0.00024944450706243515, 0.0002877325750887394, 0.0002665249921847135, 0.00031466217478737235, 0.0002523418515920639, 0.00025070441188290715, 0.0002367909182794392, 0.00024094610125757754, 0.00021231660502962768, 0.00018820451805368066, 0.00017897580983117223, 0.00021063859458081424, 0.00018122862093150616, 0.00016347685595974326 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014079716056585312, 0.014167090877890587, 0.014255538582801819, 0.014355218037962914, 0.009655896574258804, 0.009734091348946095, 0.009828771464526653, 0.009915035218000412, 0.007479202933609486, 0.007559895049780607, 0.0076358383521437645, 0.007706487085670233, 0.0062490664422512054, 0.006315518636256456, 0.0063744354993104935, 0.006439714692533016, 0.0054116579703986645, 0.005481995176523924, 0.005541813559830189, 0.005614723544567823, 0.004862982779741287, 0.004923288244754076, 0.0049925679340958595, 0.005004179198294878, 0.004448203835636377, 0.004490054678171873, 0.00455933902412653, 0.004625064320862293, 0.004160413984209299, 0.004242690280079842, 0.004270957782864571, 0.004303553607314825 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.6855086818832206e-05, "first_to_last_ratio": 1.1121706932103832, "imbalance_abs_log_ratio": 0.10631368513677814, "last_quartile_mean": 3.313797697046539e-05, "mean": 3.537216105087282e-05, "normalized": [ 1.3329674878497746, 1.065596392820356, 0.9206330800727252, 0.8842664165696186, 1.1462361065020723, 1.065158463293282, 0.99582829507555, 0.9247019781352203, 1.132583771771479, 1.005206548698436, 0.979948053013938, 0.967083333200755, 1.1057115808592233, 1.043785177992708, 0.995436955984306, 0.9279460300710988, 1.1098294348763655, 1.0288442525987258, 1.0234795644677463, 0.9449806453153974, 1.0109568163965834, 1.0260998393764513, 0.9416381672270038, 0.926379336673748, 0.9884731771330477, 0.9341156116917169, 0.9458688462068515, 0.9053302302997448, 0.9662542702797254, 0.9495526789453999, 0.9100381269887573, 0.8950693296121922 ], "peak_layer": 1, "peak_normalized": 1.3329674878497746, "population_cv": 0.09165679612351446, "rest_mean": 3.551162256001219e-05, "spike_contrast": 0.9748658722370791, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.461906890152022e-05, "top_five_layers": [ 1, 5, 9, 17, 13 ], "top_five_spike_overlap": 0 }, "values": [ 4.714994065579958e-05, 3.769244722207077e-05, 3.256478157709353e-05, 3.127841409877874e-05, 4.054484816151671e-05, 3.767695670831017e-05, 3.5224598832428455e-05, 3.270870729465969e-05, 4.006193557870574e-05, 3.555632792995311e-05, 3.4662880352698267e-05, 3.420782741159201e-05, 3.911140811396763e-05, 3.692093741847202e-05, 3.521075632306747e-05, 3.2823456422192976e-05, 3.9257065509445965e-05, 3.6392444599187e-05, 3.620268398663029e-05, 3.342600757605396e-05, 3.575972732505761e-05, 3.6295368772698566e-05, 3.330777690280229e-05, 3.276803909102455e-05, 3.49644324160181e-05, 3.3041687856893986e-05, 3.3457425161032006e-05, 3.202348671038635e-05, 3.417850166442804e-05, 3.358773028594442e-05, 3.2190015190280974e-05, 3.166053647873923e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.015105893515283e-05, "first_to_last_ratio": 0.6588274570259804, "imbalance_abs_log_ratio": 0.4172936042156455, "last_quartile_mean": 0.00010647865110513521, "mean": 9.056432463694364e-05, "normalized": [ 0.8598786252117567, 0.7452797309823858, 0.6694431442719111, 0.6417832925446045, 0.8822867975473095, 0.8318257491363829, 0.804532248804658, 0.7617644993607329, 0.969439786050785, 0.9144645011121496, 0.8951531228323205, 0.876468558935897, 1.0407786059783084, 1.0056397226462848, 0.9828357943952414, 0.9490839045924557, 1.1213662721801265, 1.0774120583919133, 1.0554308943167812, 1.0307906296953837, 1.1314055055223455, 1.1556700189426168, 1.1045075200854564, 1.0869669205578247, 1.2145839027446308, 1.1552275050200584, 1.1514380979421175, 1.1280886596086177, 1.2254687972249847, 1.20187962367999, 1.1639632986601018, 1.165142211023867 ], "peak_layer": 29, "peak_normalized": 1.2254687972249847, "population_cv": 0.1618495930711073, "rest_mean": 8.82393912804589e-05, "spike_contrast": 1.1686273359956587, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010311896476196125, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 7.787432696204633e-05, 6.74957555020228e-05, 6.062766624381766e-05, 5.812267045257613e-05, 7.990370795596391e-05, 7.533373718615621e-05, 7.286191976163536e-05, 6.898868741700426e-05, 8.779665949987248e-05, 8.281785994768143e-05, 8.106893801596016e-05, 7.937678310554475e-05, 9.425741154700518e-05, 9.107508230954409e-05, 8.900985994841903e-05, 8.595314284320921e-05, 0.00010155577911064029, 9.757509542396292e-05, 9.558438614476472e-05, 9.335285722045228e-05, 0.00010246497549815103, 0.00010466247476870194, 0.00010002897761296481, 9.844042506301776e-05, 0.00010999797086697072, 0.00010462239879416302, 0.00010427921370137483, 0.00010216458758804947, 0.00011098375398432836, 0.00010884741641348228, 0.00010541355004534125, 0.00010552031744737178 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06221066089347005, "first_to_last_ratio": 0.6588274447143223, "imbalance_abs_log_ratio": 0.4172936229028725, "last_quartile_mean": 0.09442633483558893, "mean": 0.0803133494919166, "normalized": [ 0.8598786809861813, 0.7452797356777653, 0.6694431594543391, 0.6417832788211475, 0.8822868217673959, 0.83182573854958, 0.804532202278358, 0.7617644434307961, 0.9694397651274825, 0.9144645475330049, 0.8951531251987148, 0.8764685422804359, 1.0407786734529123, 1.0056396718254574, 0.9828357789429103, 0.9490838578168909, 1.121366176654653, 1.0774119964231388, 1.0554309636293913, 1.030790617723408, 1.131405440813584, 1.1556700726993148, 1.1045075638748008, 1.0869669141870728, 1.214583886452764, 1.155227565075462, 1.151438141297865, 1.1280886745503538, 1.2254688318483915, 1.20187965027907, 1.1639633335063522, 1.1651421478410058 ], "peak_layer": 29, "peak_normalized": 1.2254688318483915, "population_cv": 0.16184959877781283, "rest_mean": 0.07825157529226055, "spike_contrast": 1.1686273385361958, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0914469301700592, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.06905973702669144, 0.05985591188073158, 0.05376522243022919, 0.051543764770030975, 0.0708594098687172, 0.06680671125650406, 0.0646146759390831, 0.06117985397577286, 0.07785895466804504, 0.0734437108039856, 0.07189274579286575, 0.07039212435483932, 0.08358842134475708, 0.08076629042625427, 0.07893483340740204, 0.07622410356998444, 0.09006067365407944, 0.08653056621551514, 0.08476519584655762, 0.08278624713420868, 0.09086696058511734, 0.09281573444604874, 0.08870670199394226, 0.0872979536652565, 0.09754730015993118, 0.09278019517660141, 0.09247585386037827, 0.09060057997703552, 0.09842150658369064, 0.09652698040008545, 0.09348179399967194, 0.09357646852731705 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.077007694533677e-06, "first_to_last_ratio": 0.4632676420751026, "imbalance_abs_log_ratio": 0.7694503312712233, "last_quartile_mean": 1.5276283193088602e-05, "mean": 1.1745167029175718e-05, "normalized": [ 0.5860812018784138, 0.5338863181232825, 0.4896831424227033, 0.48432444022666415, 0.7227742595277804, 0.658356157633798, 0.6679626701195096, 0.6773027265164295, 0.9317627933811012, 0.8396840560424388, 0.8002283441953012, 0.8237560838623014, 0.9947731150675175, 0.9485376018841178, 1.012754293639493, 0.980582798985168, 1.1861357451339187, 1.074396404738039, 1.1583203932505854, 1.1590437197037688, 1.1716087388237089, 1.3380122311407743, 1.182136735603702, 1.17274255168395, 1.3256589211015541, 1.1785808128912905, 1.3283612705891032, 1.2366027296837898, 1.3388375403515687, 1.3849270873253279, 1.3156503626951777, 1.2965347517777213 ], "peak_layer": 30, "peak_normalized": 1.3849270873253279, "population_cv": 0.2782190894097951, "rest_mean": 1.1227440423950881e-05, "spike_contrast": 1.2951207174852213, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4540890697389841e-05, "top_five_layers": [ 30, 29, 22, 27, 25 ], "top_five_spike_overlap": 2 }, "values": [ 6.883621608722024e-06, 6.2705839809495956e-06, 5.751410299126292e-06, 5.6884714467742015e-06, 8.48910440254258e-06, 7.732503036095295e-06, 7.845333129807841e-06, 7.955033652251586e-06, 1.0943709639832377e-05, 9.862229489954188e-06, 9.39881556405453e-06, 9.67515279626241e-06, 1.168377639260143e-05, 1.1140732567582745e-05, 1.1894968338310719e-05, 1.1517108760017436e-05, 1.3931362445873674e-05, 1.2618965229194146e-05, 1.3604666492028628e-05, 1.3613162082037888e-05, 1.3760740330326371e-05, 1.5715177141828462e-05, 1.3884393410990015e-05, 1.377405715174973e-05, 1.5570085452054627e-05, 1.38426285047899e-05, 1.56018249981571e-05, 1.4524105608870741e-05, 1.572487053635996e-05, 1.62661999638658e-05, 1.5452533261850476e-05, 1.5228017218760215e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.028231200616574e-05, "first_to_last_ratio": 0.5725175422355306, "imbalance_abs_log_ratio": 0.5577119025637401, "last_quartile_mean": 0.00010529338851483772, "mean": 8.693937525094952e-05, "normalized": [ 0.6520176082954547, 0.61792639701702, 0.5860559002851884, 0.5711155588169554, 0.8204737244004536, 0.784378974164202, 0.772878878140371, 0.7422194596148288, 0.9603819210570083, 0.917062379210463, 0.9022792075902993, 0.8821977885039085, 1.057360487646653, 1.0255127545988867, 1.0067830966339102, 0.9749202156934387, 1.1510091959216098, 1.1091409226242501, 1.0872456061539109, 1.0647793564016073, 1.1683829082644246, 1.1939706281941995, 1.1404893610428462, 1.1225154239128359, 1.2535795945499137, 1.1915479696415308, 1.1872717446378291, 1.1619194447200984, 1.2611193116864508, 1.2370940006457103, 1.1981927894482458, 1.1981773904854955 ], "peak_layer": 29, "peak_normalized": 1.2611193116864508, "population_cv": 0.20676778332368514, "rest_mean": 8.410921549916061e-05, "spike_contrast": 1.215351222858925, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001022222379106097, "top_five_layers": [ 29, 25, 30, 31, 32 ], "top_five_spike_overlap": 1 }, "values": [ 5.6686003517825156e-05, 5.3722134907729924e-05, 5.095133383292705e-05, 4.965242987964302e-05, 7.133147300919518e-05, 6.81934179738164e-05, 6.719360681017861e-05, 6.452809611801058e-05, 8.349500421900302e-05, 7.972883031470701e-05, 7.844359060982242e-05, 7.669772458029911e-05, 9.192626021103933e-05, 8.915743819670752e-05, 8.75290934345685e-05, 8.475895447190851e-05, 0.00010006802040152252, 9.642801887821406e-05, 9.452445374336094e-05, 9.257125202566385e-05, 0.00010157848009839654, 0.00010380306048318744, 9.915343252941966e-05, 9.759078966453671e-05, 0.00010898542677750811, 0.00010359243606217206, 0.00010322066373191774, 0.00010101655061589554, 0.00010964092507492751, 0.0001075521795428358, 0.00010417013254482299, 0.00010416879376862198 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.289650744089158e-05, "first_to_last_ratio": 0.5036607689093815, "imbalance_abs_log_ratio": 0.6858523150899144, "last_quartile_mean": 0.00010502407712920103, "mean": 8.374395429200376e-05, "normalized": [ 0.5462883602941595, 0.5343271256807102, 0.5249643991287053, 0.5130648517134513, 0.7580131602578075, 0.7348274620741431, 0.7341801809099809, 0.7074999463902408, 0.9290893112957194, 0.896130362600229, 0.8896360907643062, 0.8667732512123232, 1.0501229826066591, 1.0276121076179003, 1.01304723882372, 0.9917393510831477, 1.161342307568102, 1.1291245609681106, 1.1079392656735292, 1.0938109858524918, 1.2035226235600145, 1.226263478849651, 1.1724755435916903, 1.1553301515952579, 1.2932755854998876, 1.227360989677261, 1.2282904680523223, 1.203004103561663, 1.301298744133899, 1.2818290480161694, 1.2464280682225441, 1.251387892724203 ], "peak_layer": 29, "peak_normalized": 1.301298744133899, "population_cv": 0.24714813814810152, "rest_mean": 8.048455465878187e-05, "spike_contrast": 1.2591821218500583, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010134471231140196, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 4.574834747472778e-05, 4.474666638998315e-05, 4.396259464556351e-05, 4.296607949072495e-05, 6.347901944536716e-05, 6.153735739644617e-05, 6.14831515122205e-05, 5.924884317209944e-05, 7.780561281833798e-05, 7.504550012527034e-05, 7.450164412148297e-05, 7.258701953105628e-05, 8.794145105639473e-05, 8.60563013702631e-05, 8.483658166369423e-05, 8.305217488668859e-05, 9.725539712235332e-05, 9.455735562369227e-05, 9.278321522288024e-05, 9.160005720332265e-05, 0.00010078774357680231, 0.00010269215272273868, 9.818773833103478e-05, 9.675191540736705e-05, 0.00010830401151906699, 0.00010278406261932105, 0.00010286190081387758, 0.00010074432066176087, 0.00010897590254899114, 0.00010734543320722878, 0.00010438081517349929, 0.00010479617048986256 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010133173600479495, "first_to_last_ratio": 0.7712367006907566, "imbalance_abs_log_ratio": 0.25975994774816064, "last_quartile_mean": 0.00013138863323547412, "mean": 0.00011757463107642252, "normalized": [ 0.9240254436843522, 0.8516610763218788, 0.7400083948384197, 0.7305203183107621, 0.9945100925181989, 0.9162256174012862, 0.9029060711458456, 0.8349457218511338, 1.057384774375838, 0.9561549232760007, 0.9255087968738563, 0.9289705112696707, 1.0651337322266232, 1.006401362110571, 0.966928758219108, 0.9306903222204219, 1.1004660046561014, 1.0434523934672617, 1.009934985409439, 0.9657192405683421, 1.061624429236619, 1.0862247001295862, 1.0383026763618062, 1.022368851612807, 1.1429833503196458, 1.0926891991234107, 1.0888718388877934, 1.0734199646869733, 1.177032649101865, 1.140664442905499, 1.1108444025161495, 1.1134249543727335 ], "peak_layer": 29, "peak_normalized": 1.177032649101865, "population_cv": 0.10936728326378568, "rest_mean": 0.00011604396611262389, "spike_contrast": 1.0844184846181808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012584022188093512, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.00010864195064641535, 0.00010013373685069382, 8.700621401658282e-05, 8.58906569192186e-05, 0.00011692915722960606, 0.00010772488894872367, 0.0001061588482116349, 9.816843521548435e-05, 0.0001243216247530654, 0.00011241956235608086, 0.00010881635535042733, 0.00010922336514340714, 0.00012523270561359823, 0.00011832726886495948, 0.00011368629202479497, 0.00010942557128146291, 0.0001293868845095858, 0.00012268353020772338, 0.00011874273332068697, 0.00011354408343322575, 0.00012482010060921311, 0.00012771246838383377, 0.0001220780541189015, 0.00012020464055240154, 0.00013438584574032575, 0.0001284725294681266, 0.00012802370474673808, 0.0001262069563381374, 0.00013838917948305607, 0.00013411320105660707, 0.00013060712080914527, 0.00013091052824165672 ] } } } } }, "uniform_value_backward": { "bits_per_byte": 8.06433666787865, "forward": { "activation_sha256": { "attention_branch_output": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", "mlp_branch_output": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", "post_attention_state": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", "post_mlp_state": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", "pre_attention_input": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", "pre_mlp_input": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248" }, "logits_sha256": "30ae47372d19e5c17da2b2a4846b2d27ef6e44fdc06045752d040a56bc6e9398", "loss_nats": 5.5897722244262695, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.5897722244262695, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_value_backward", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0021227982360869646, 0.00224129855632782, 0.0024903982412070036, 0.0023722455371171236, 0.0023577234242111444, 0.0023993616923689842, 0.002688874490559101, 0.0029664887115359306, 0.0029547461308538914, 0.003052416956052184, 0.0028214550111442804, 0.0032544767018407583, 0.00348988501355052, 0.003914494998753071, 0.0035071265883743763, 0.004211232997477055, 0.004072706215083599, 0.004050548188388348, 0.003929397091269493, 0.004410331137478352, 0.004824656527489424, 0.00433384021744132, 0.004475373309105635, 0.004755842965096235, 0.004843301139771938, 0.0050001065246760845, 0.005346097983419895, 0.004958339501172304, 0.0054801348596811295, 0.005455824080854654, 0.005286100786179304, 0.005402065813541412 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015727665777376387, "first_to_last_ratio": 3.444367030996094, "imbalance_abs_log_ratio": 1.2367401520242614, "last_quartile_mean": 4.566199140754179e-05, "mean": 8.846047569477378e-05, "normalized": [ 2.4938583213844665, 2.2321521039252294, 1.9412946714691228, 1.7875630514690435, 1.605255150569201, 1.4809330135294976, 1.3782219007164203, 1.3041730210393685, 1.1983684502302745, 1.1165155187552953, 1.0705163494358503, 1.028915549335682, 0.969265519878856, 0.9296915475356099, 0.8760186941641476, 0.840544602365157, 0.8215144482528359, 0.7958412144142111, 0.7578169321029499, 0.7149281644739428, 0.6789722877716906, 0.648159768212111, 0.6127699836305072, 0.5872278577962867, 0.5712871725092085, 0.5525140937217092, 0.5351005407841478, 0.5196042162209058, 0.49974197043571306, 0.49405312503381205, 0.48454882285128825, 0.4726319359854579 ], "peak_layer": 1, "peak_normalized": 2.4938583213844665, "population_cv": 0.5254866190950809, "rest_mean": 9.469065830840177e-05, "spike_contrast": 0.5789112734082525, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.48174895811826e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00022060789342503995, 0.0001974572369363159, 0.00017172785010188818, 0.00015812867786735296, 0.00014200163423083723, 0.0001310040388489142, 0.00012191816495032981, 0.00011536776582943276, 0.00010600824316497892, 9.876749390969053e-05, 9.469838551012799e-05, 9.101835894398391e-05, 8.574168896302581e-05, 8.224095654441044e-05, 7.749303040327504e-05, 7.435497536789626e-05, 7.267155888257548e-05, 7.040049240458757e-05, 6.703684630338103e-05, 6.324288551695645e-05, 6.0062211559852585e-05, 5.733652142225765e-05, 5.42059242434334e-05, 5.1946455641882494e-05, 5.053633503848687e-05, 4.887565955868922e-05, 4.733524838229641e-05, 4.596443613991141e-05, 4.420741242938675e-05, 4.370417445898056e-05, 4.286341936676763e-05, 4.180924588581547e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00022213962802197784, "first_to_last_ratio": 1.7678973829107683, "imbalance_abs_log_ratio": 0.5697909211909248, "last_quartile_mean": 0.00012565187898871955, "mean": 0.00016188367408176418, "normalized": [ 1.7325269150218519, 1.5951124185334453, 1.452057969341735, 1.3730142236731557, 1.2857984704138503, 1.2257558334519054, 1.176224442377172, 1.1372505911497117, 1.088955563427776, 1.050933480062714, 1.0259126283978994, 1.0049209692997896, 0.9780904283068869, 0.9585002626676595, 0.9364797090707767, 0.9194681619525875, 0.9067005574410794, 0.8936181540011504, 0.876961137714969, 0.8605080845266135, 0.8461083257765747, 0.8336465310750286, 0.8211746685608088, 0.8107906186997867, 0.8025238658005067, 0.7926026656489187, 0.7851375623337778, 0.7775608141673811, 0.7689788128837574, 0.7655650153667295, 0.7607140375905382, 0.7564070812634623 ], "peak_layer": 1, "peak_normalized": 1.7325269150218519, "population_cv": 0.2510633514944888, "rest_mean": 0.00016719439386010722, "spike_contrast": 0.7967120440064878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001332057872787118, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002804678224492818, 0.0002582226588856429, 0.00023506447905674577, 0.00022226858709473163, 0.00020814978051930666, 0.00019842985784634948, 0.00019041153427679092, 0.00018410230404697359, 0.00017628412751946598, 0.0001701289729680866, 0.0001660785055719316, 0.0001626802986720577, 0.0001583368721185252, 0.00015516554412897676, 0.00015160077600739896, 0.00014884688425809145, 0.0001467800175305456, 0.00014466218999587, 0.00014196569100022316, 0.0001393022103002295, 0.00013697112444788218, 0.00013495376333594322, 0.0001329347724094987, 0.0001312537642661482, 0.00012991551193408668, 0.00012830943160224706, 0.0001271009532501921, 0.00012587440141942352, 0.0001244851155206561, 0.00012393247743602842, 0.0001231471833307296, 0.00012244995741639286 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19699564762413502, "first_to_last_ratio": 1.7678974461601602, "imbalance_abs_log_ratio": 0.5697909569675436, "last_quartile_mean": 0.11142934113740921, "mean": 0.14356006286107004, "normalized": [ 1.73252692376823, 1.5951124529267644, 1.4520579427898739, 1.3730142509369438, 1.2857984860257277, 1.2257558423238033, 1.1762244498747965, 1.1372506023188744, 1.0889555406950477, 1.0509335152923371, 1.0259126331612176, 1.004920958483027, 0.9780904724894115, 0.9585002702989701, 0.9364797557971557, 0.9194681924019411, 0.9067005932791752, 0.893618176643793, 0.876961081367795, 0.8605080517791706, 0.8461083423470157, 0.8336465303631113, 0.8211746500314575, 0.8107906024915998, 0.8025238662049899, 0.792602599370704, 0.7851375422989607, 0.7775607992253795, 0.7689787783378558, 0.7655650375651303, 0.7607140142018359, 0.7564070449079043 ], "peak_layer": 1, "peak_normalized": 1.73252692376823, "population_cv": 0.25106336091926634, "rest_mean": 0.1482696624265777, "spike_contrast": 0.7967120399011176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1181282252073288, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2487216740846634, 0.2289944440126419, 0.20845752954483032, 0.19711001217365265, 0.18458931148052216, 0.17596958577632904, 0.1688588559627533, 0.16326376795768738, 0.15633052587509155, 0.15087208151817322, 0.14728008210659027, 0.1442665159702301, 0.14041472971439362, 0.13760235905647278, 0.1344410926103592, 0.1319989114999771, 0.13016599416732788, 0.1282878816127777, 0.12589658796787262, 0.12353459000587463, 0.12146736681461334, 0.11967834830284119, 0.11788788437843323, 0.11639714986085892, 0.11521037667989731, 0.11378607898950577, 0.11271439492702484, 0.11162667721509933, 0.11039464175701141, 0.10990456491708755, 0.10920815169811249, 0.10858984291553497 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.813340178268845e-05, "first_to_last_ratio": 2.0396781258635253, "imbalance_abs_log_ratio": 0.7127920139678758, "last_quartile_mean": 3.3403996894776355e-05, "mean": 4.817690887648496e-05, "normalized": [ 1.6677253734616262, 1.5880838657379246, 1.4689679818917765, 1.414042908307037, 1.3674754530876347, 1.307279124984285, 1.2642920390991146, 1.2360018052494937, 1.16456296339031, 1.101785452839103, 1.0797656603944665, 1.0672809591929815, 1.0128091896347613, 0.9955804650846075, 0.9623876307655963, 0.9380984486976066, 0.9310658554452689, 0.9082443423860135, 0.8811967490595315, 0.8644115118569954, 0.8497936463301483, 0.8309315019236491, 0.7889969150425813, 0.7623309639866356, 0.7584133539756696, 0.7445182980676538, 0.7180454335344151, 0.7000361319857387, 0.6663708133598921, 0.6665317313809671, 0.6558653811637546, 0.6371080486827607 ], "peak_layer": 1, "peak_normalized": 1.6677253734616262, "population_cv": 0.28258812904813146, "rest_mean": 4.997824995640527e-05, "spike_contrast": 0.7693279992487513, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.844966704491526e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.034585334826261e-05, 7.650897168787196e-05, 7.077033660607412e-05, 6.81242163409479e-05, 6.588074029423296e-05, 6.298066728049889e-05, 6.090968236094341e-05, 5.954674634267576e-05, 5.610504376818426e-05, 5.308061736286618e-05, 5.201977182878181e-05, 5.1418297516647726e-05, 4.879401603830047e-05, 4.796398934558965e-05, 4.6364861191250384e-05, 4.519468348007649e-05, 4.4855874875793234e-05, 4.3756404920713976e-05, 4.245333548169583e-05, 4.1644674638519064e-05, 4.094043106306344e-05, 4.0031711250776425e-05, 3.8011432479834184e-05, 3.672674938570708e-05, 3.653801104519516e-05, 3.586859020288102e-05, 3.459320942056365e-05, 3.372557694092393e-05, 3.210368595318869e-05, 3.21114384860266e-05, 3.159756670356728e-05, 3.069389640586451e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018498800636734813, "first_to_last_ratio": 1.4771173407265819, "imbalance_abs_log_ratio": 0.39009244570614415, "last_quartile_mean": 0.0001252358233614359, "mean": 0.00015057798327688943, "normalized": [ 1.36489476841379, 1.32224224178262, 1.270236278236952, 1.2358216684832493, 1.2002806183282966, 1.1699184301039023, 1.1431585124660382, 1.1216044194279113, 1.0931779382416982, 1.0676137521656621, 1.050334546837667, 1.0352083927879365, 1.0149316924423049, 1.001461761419052, 0.9840691969120363, 0.9695910166861202, 0.9584136856662008, 0.9467149797350989, 0.9322036518549593, 0.9180752132052378, 0.9048391519289843, 0.8925982934025656, 0.8798985862981495, 0.8691051109125614, 0.860352873872478, 0.849675270316085, 0.8415818308236442, 0.8332603200076834, 0.823928337260059, 0.8201054366601405, 0.8147645087933348, 0.8099375145275801 ], "peak_layer": 1, "peak_normalized": 1.36489476841379, "population_cv": 0.15663918075507588, "rest_mean": 0.0001538862707093358, "spike_contrast": 0.8624111204329009, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000132713231141679, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00020552310161292553, 0.00019910057017114013, 0.00019126961706206203, 0.0001860875345300883, 0.00018073583487421274, 0.00017616395780351013, 0.0001721345033729449, 0.00016888893151190132, 0.0001646085293032229, 0.00016075912571977824, 0.00015815725782886147, 0.0001558795920573175, 0.00015282636741176248, 0.00015079809236340225, 0.0001481791550759226, 0.00014599905989598483, 0.00014431599993258715, 0.00014255443238653243, 0.00014036934589967132, 0.000138241914100945, 0.00013624885468743742, 0.00013440565089695156, 0.0001324933546129614, 0.0001308680948568508, 0.00012955020065419376, 0.00012794238864444196, 0.0001267236948478967, 0.00012547065853141248, 0.00012406546738930047, 0.00012348982272669673, 0.0001226855965796858, 0.00012195875751785934 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015959660413500387, "first_to_last_ratio": 1.2728751185715401, "imbalance_abs_log_ratio": 0.2412782146622287, "last_quartile_mean": 0.00012538276678242255, "mean": 0.00014144585429676226, "normalized": [ 1.173955566226187, 1.1612997099437044, 1.147257651804958, 1.1326524298721496, 1.1221543736292459, 1.1091111750587395, 1.0952125313023318, 1.0849401932479867, 1.070907908686229, 1.0558579375605581, 1.0430997159190893, 1.0346471150202732, 1.0228091528121042, 1.0150746526225596, 1.0038187840838828, 0.9945868690199192, 0.986355871119351, 0.9777591356798918, 0.9676015085554662, 0.9595938626090281, 0.9511867345631487, 0.9428977124791953, 0.9335239262292595, 0.9222035520824579, 0.9142471402553919, 0.9038764491526137, 0.8952592406413572, 0.8872171299760644, 0.877779969795339, 0.8748896658804006, 0.8709399062198925, 0.8672824279512239 ], "peak_layer": 1, "peak_normalized": 1.173955566226187, "population_cv": 0.09275081554992197, "rest_mean": 0.00014320575994335942, "spike_contrast": 0.9213481626529775, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013194236380513756, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016605114797130227, 0.0001642610295675695, 0.0001622748386580497, 0.00016020899056456983, 0.00015872408403083682, 0.00015687917766626924, 0.0001549132721265778, 0.00015346029249485582, 0.00015147548401728272, 0.00014934672799427062, 0.00014754213043488562, 0.000146346545079723, 0.00014467211440205574, 0.00014357810141518712, 0.00014198600547388196, 0.00014068018936086446, 0.00013951594883110374, 0.00013829997624270618, 0.00013686322199646384, 0.0001357305736746639, 0.00013454142026603222, 0.0001333689724560827, 0.00013204308925196528, 0.00013044186925981194, 0.00012931646779179573, 0.00012784957652911544, 0.00012663070810958743, 0.000125493184896186, 0.0001241583377122879, 0.00012374951620586216, 0.0001231908390764147, 0.00012267350393813103 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031297675377572887, "first_to_last_ratio": 2.208693462790297, "imbalance_abs_log_ratio": 0.7924011473984336, "last_quartile_mean": 0.00014170221402309835, "mean": 0.00020716176595669822, "normalized": [ 1.8096319791310764, 1.7504284704923412, 1.6109023900661548, 1.5434988529701714, 1.4353862656069223, 1.3677596328902204, 1.3131374518280827, 1.2555297512219317, 1.1870915168673883, 1.122656500215855, 1.088514230207685, 1.04408820740568, 0.9901334868561585, 0.9607872809839492, 0.9219812869372472, 0.880015778430751, 0.8614610505244651, 0.8397919049431777, 0.8116424497290868, 0.7845312039842096, 0.7619415782348183, 0.7436794877201931, 0.7280047725498856, 0.715266757195506, 0.7065993237444729, 0.696468071137219, 0.6907565141722826, 0.6863265628748565, 0.6782234011874358, 0.6754056248242399, 0.6708832321424102, 0.6674749829241257 ], "peak_layer": 1, "peak_normalized": 1.8096319791310764, "population_cv": 0.33826602846938675, "rest_mean": 0.00021747771663487784, "spike_contrast": 0.6964190843920172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001514556322945282, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00037488655652850866, 0.0003626218531280756, 0.0003337173839099705, 0.0003197539481334388, 0.0002973571536131203, 0.00028334750095382333, 0.00027203187346458435, 0.0002600977604743093, 0.0002459199749864638, 0.00023257150314748287, 0.00022549853019881994, 0.0002162951568607241, 0.00020511780166998506, 0.00019903838983736932, 0.00019099927158094943, 0.00018230562272947282, 0.00017846179252956063, 0.00017397277406416833, 0.0001681412832112983, 0.0001625248696655035, 0.0001578451629029587, 0.00015406195598188788, 0.00015081475430633873, 0.0001481759245507419, 0.00014638036373071373, 0.0001442815555492416, 0.0001430983393220231, 0.00014218062278814614, 0.00014050195750314742, 0.00013991822197567672, 0.00013898135512135923, 0.00013827529619447887 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027448106557130814, 0.0027243648655712605, 0.0027103275060653687, 0.0027234479784965515, 0.0027595730498433113, 0.0027272943407297134, 0.002728298306465149, 0.0027007071767002344, 0.002684502862393856, 0.002683063969016075, 0.0027139110025018454, 0.002666700165718794, 0.002679243916645646, 0.0026584581937640905, 0.002684473292902112, 0.0026407993864268064, 0.002663099905475974, 0.0026283918414264917, 0.002597011858597398, 0.0026434636674821377, 0.0025708195753395557, 0.002691218862310052, 0.0026598076801747084, 0.0026596153620630503, 0.0026765079237520695, 0.002582729561254382, 0.002614464145153761, 0.0025760054122656584, 0.002634295728057623, 0.00259007909335196, 0.00257018837146461, 0.0025643527042120695 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.000156725715896755, "first_to_last_ratio": 3.4451295459637388, "imbalance_abs_log_ratio": 1.236961507811942, "last_quartile_mean": 4.549196592051885e-05, "mean": 8.812961027615529e-05, "normalized": [ 2.4952457548653, 2.231881013356295, 1.939989240144028, 1.7866526882708733, 1.6079164676041713, 1.4797016884434102, 1.3811975174028512, 1.3042540698565042, 1.1954344722384815, 1.1168849900912188, 1.0692220969851183, 1.0304138184362992, 0.9676518223611118, 0.9260610329319834, 0.8738547948342114, 0.8395441231817105, 0.8221010666653766, 0.7971164256793037, 0.7591159941756684, 0.7162115141810864, 0.6791638220644127, 0.6489084022271779, 0.6150954832405342, 0.5868306199881643, 0.5699702288291514, 0.552179306852942, 0.5335984532948914, 0.5198944860736878, 0.5024723171712969, 0.4942325662695092, 0.4850276094601851, 0.47217611282304317 ], "peak_layer": 1, "peak_normalized": 2.4952457548653, "population_cv": 0.5254777238950282, "rest_mean": 9.433142622583546e-05, "spike_contrast": 0.5792322488273548, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.463980414788239e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021990503591950983, 0.00019669480388984084, 0.00017097049567382783, 0.00015745700511615723, 0.00014170505164656788, 0.0001304055331274867, 0.00012172439892310649, 0.00011494340287754312, 0.00010535317414905876, 9.843063890002668e-05, 9.423012670595199e-05, 9.080996824195608e-05, 8.527877798769623e-05, 8.16133979242295e-05, 7.701248250668868e-05, 7.398869638564065e-05, 7.24514466128312e-05, 7.024955993983895e-05, 6.690059672109783e-05, 6.311944162007421e-05, 5.985444295220077e-05, 5.718804459320381e-05, 5.420812522061169e-05, 5.1717153837671503e-05, 5.023125413572416e-05, 4.866334711550735e-05, 4.7025823732838035e-05, 4.581809844239615e-05, 4.428268948686309e-05, 4.3556523451115936e-05, 4.274529419490136e-05, 4.161269680480473e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002203928706876468, "first_to_last_ratio": 1.7624395804596433, "imbalance_abs_log_ratio": 0.5666989745264205, "last_quartile_mean": 0.00012504988717410015, "mean": 0.00016078790008577926, "normalized": [ 1.7301295632680096, 1.5926765618359837, 1.4498368649995454, 1.3715838442653223, 1.2853282969253275, 1.2242662006350955, 1.1759759999659367, 1.1358472505674493, 1.0864560712260232, 1.0500344477950443, 1.0256300088344572, 1.0049332388641516, 0.9775387355078093, 0.9578140668049778, 0.9358065303644342, 0.9184543281568608, 0.9072826301440372, 0.8934865935730769, 0.8779252804708734, 0.8613603707803614, 0.8467225584074481, 0.8352602527072714, 0.8224752340358507, 0.8113194646910352, 0.80159772798593, 0.7940537841119606, 0.785769247679408, 0.7781743501685717, 0.7724121545166266, 0.7676777199663887, 0.7637141967462265, 0.7584564239985051 ], "peak_layer": 1, "peak_normalized": 1.7301295632680096, "population_cv": 0.2498913679417364, "rest_mean": 0.0001660440253477578, "spike_contrast": 0.7974079368034485, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013240482367109506, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027818389935418963, 0.00025608311989344656, 0.00023311622499022633, 0.00022053408611100167, 0.00020666523778345436, 0.0001968471915461123, 0.00018908271158579737, 0.0001826304942369461, 0.0001746889902278781, 0.00016883283387869596, 0.00016490889538545161, 0.00016158110520336777, 0.00015717640053480864, 0.00015400491247419268, 0.0001504663669038564, 0.00014767634274903685, 0.00014588006888516247, 0.00014366183313541114, 0.00014115976227913052, 0.00013849632523488253, 0.00013614274212159216, 0.0001342997420579195, 0.00013224406575318426, 0.00013045035302639008, 0.00012888721539638937, 0.00012767424050252885, 0.0001263421872863546, 0.0001251210196642205, 0.00012419452832546085, 0.00012343328853603452, 0.00012279600196052343, 0.00012195061572128907 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19544659927487373, "first_to_last_ratio": 1.7624395617194093, "imbalance_abs_log_ratio": 0.5666989638932991, "last_quartile_mean": 0.11089549027383327, "mean": 0.14258831785991788, "normalized": [ 1.7301295593799786, 1.5926765086027146, 1.4498368632986096, 1.3715838909156086, 1.2853282557768668, 1.224266111161914, 1.1759760185820738, 1.1358472270183295, 1.0864560720464596, 1.050034483149275, 1.025630003735948, 1.0049332495130952, 0.9775386792530747, 0.9578141330250067, 0.9358065753109587, 0.9184542876534186, 0.9072827263675407, 0.8934866319653527, 0.8779253472761699, 0.8613603464001749, 0.8467225704769723, 0.8352602246328348, 0.8224752140046494, 0.8113194329409806, 0.8015977187598701, 0.7940537797484172, 0.7857692153409586, 0.7781743825002613, 0.7724121455314925, 0.7676776608495898, 0.7637142124554999, 0.758456472325904 ], "peak_layer": 1, "peak_normalized": 1.7301295593799786, "population_cv": 0.24989136150327743, "rest_mean": 0.14724950271624107, "spike_contrast": 0.7974079196861148, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11741791963577271, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24669626355171204, 0.2270970642566681, 0.20672979950904846, 0.19557183980941772, 0.18327279388904572, 0.17456604540348053, 0.16768044233322144, 0.16195854544639587, 0.1549159437417984, 0.1497226506471634, 0.14624285697937012, 0.14329174160957336, 0.13938559591770172, 0.13657310605049133, 0.13343508541584015, 0.1309608519077301, 0.1293679177761078, 0.12740075588226318, 0.1251818984746933, 0.1228199228644371, 0.12073274701833725, 0.11909835040569305, 0.11727535724639893, 0.11568467319011688, 0.11429847031831741, 0.11322279274463654, 0.11204151064157486, 0.11095857620239258, 0.11013694852590561, 0.10946186631917953, 0.10889672487974167, 0.108147032558918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.812671563238837e-05, "first_to_last_ratio": 2.0429267595554257, "imbalance_abs_log_ratio": 0.7143834657339574, "last_quartile_mean": 3.334760549478233e-05, "mean": 4.8136708642232406e-05, "normalized": [ 1.6706878479333491, 1.5890573277727331, 1.4709453763829752, 1.4135957551070697, 1.368779351628352, 1.3072249852707734, 1.2659085053079855, 1.2360067186760548, 1.1647282180273046, 1.1006681301372137, 1.0750864931045292, 1.0685694254166505, 1.0112110373273602, 0.9945647481091522, 0.9602230239537092, 0.9352227929155844, 0.93139607887314, 0.9092220870879134, 0.8814974932144994, 0.8653605860843365, 0.8506124631158822, 0.8320765477333729, 0.7932580049774197, 0.7619473309737773, 0.7566723544753674, 0.7419402536141336, 0.7141691805145849, 0.7010673290630277, 0.6716446941281601, 0.6659218543873533, 0.6557479674543484, 0.6349860372318865 ], "peak_layer": 1, "peak_normalized": 1.6706878479333491, "population_cv": 0.2830124551063321, "rest_mean": 4.992923641111702e-05, "spike_contrast": 0.7702312603701833, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.845705869025551e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.04214141680859e-05, 7.649198960280046e-05, 7.080646901158616e-05, 6.804584700148553e-05, 6.588853284483775e-05, 6.292550824582577e-05, 6.093666888773441e-05, 5.949729529675096e-05, 5.6066182878566906e-05, 5.298254109220579e-05, 5.1751125283772126e-05, 5.143741509527899e-05, 4.867637107963674e-05, 4.7875073505565524e-05, 4.622197593562305e-05, 4.501854709815234e-05, 4.483434167923406e-05, 4.376695869723335e-05, 4.24323879997246e-05, 4.165561040281318e-05, 4.094568430446088e-05, 4.0053426346275955e-05, 3.8184829463716596e-05, 3.667763667181134e-05, 3.642371666501276e-05, 3.571456181816757e-05, 3.4377753763692454e-05, 3.3747073757695034e-05, 3.2330764952348545e-05, 3.205528628313914e-05, 3.156554885208607e-05, 3.056613786611706e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018337754227104597, "first_to_last_ratio": 1.4712162101549686, "imbalance_abs_log_ratio": 0.38608941256902984, "last_quartile_mean": 0.00012464350311347516, "mean": 0.00014952537844692415, "normalized": [ 1.3620834570997993, 1.3194965005671844, 1.2677944868152182, 1.233561928872588, 1.1984447257225705, 1.1684695593357564, 1.141528073961558, 1.1198008373135866, 1.0910012077210556, 1.066096352737946, 1.0498630645220353, 1.0350823851472097, 1.0145919973375974, 1.0013407128400098, 0.9836056688270479, 0.9685428567287663, 0.9591027483551056, 0.9465833155582388, 0.9334001555540693, 0.9191584382428863, 0.9057431683575706, 0.8945627711391904, 0.8815040843165663, 0.8698871064160805, 0.8596264871795505, 0.8514885298372257, 0.8425068993042694, 0.8341757603627517, 0.8278236379550317, 0.8225948884437775, 0.8181940461689637, 0.812344147258793 ], "peak_layer": 1, "peak_normalized": 1.3620834570997993, "population_cv": 0.15505530299928885, "rest_mean": 0.00015278545470209792, "spike_contrast": 0.8634393039979241, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001319209666689858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00020366604439914227, 0.0001972982136067003, 0.0001895674504339695, 0.00018444881425239146, 0.00017919790116138756, 0.00017471585306338966, 0.0001706874172668904, 0.0001674386439844966, 0.00016313236847054213, 0.00015940846060402691, 0.00015698117204010487, 0.00015477108536288142, 0.00015170725237112492, 0.00014972584904171526, 0.0001470740098739043, 0.0001448217371944338, 0.00014341020141728222, 0.00014153822849038988, 0.00013956701150164008, 0.00013743751333095133, 0.00013543159002438188, 0.00013375983689911664, 0.00013180723180994391, 0.00013007019879296422, 0.00012853597581852227, 0.00012731914466712624, 0.0001259761629626155, 0.00012473044625949115, 0.00012378064275253564, 0.00012299881200306118, 0.00012234077439643443, 0.00012146606604801491 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015816818631719798, "first_to_last_ratio": 1.2671705792413261, "imbalance_abs_log_ratio": 0.23678652466939742, "last_quartile_mean": 0.00012481996418500785, "mean": 0.0001404537983944465, "normalized": [ 1.172032188999453, 1.1600674087975105, 1.1434054243316032, 1.1305758413418425, 1.1202630655069494, 1.1065457827105485, 1.0931998253234612, 1.0828906757132684, 1.0683311763286112, 1.0526488941703125, 1.0427563459749776, 1.0343706495522589, 1.0219941374371992, 1.0154952175292713, 1.0038357654470964, 0.9937321709742023, 0.9862865992066487, 0.9768881500283385, 0.968261257859838, 0.9607910277643221, 0.9520546236098718, 0.9457805296333641, 0.9349892993508679, 0.9232794945485685, 0.9140411172891727, 0.9051057888083044, 0.8959794100889387, 0.888100763485746, 0.8822517666505367, 0.8784085901067091, 0.87543446421392, 0.8702025472162866 ], "peak_layer": 1, "peak_normalized": 1.172032188999453, "population_cv": 0.09096684083762872, "rest_mean": 0.00014216970130628734, "spike_contrast": 0.9227558436510851, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013118792267050595, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016461637278553098, 0.0001629358739592135, 0.00016059563495218754, 0.00015879367128945887, 0.0001573452027514577, 0.00015541855827905238, 0.00015354406787082553, 0.00015209610864985734, 0.00015005117165856063, 0.0001478485355619341, 0.0001464590895920992, 0.00014528128667734563, 0.00014354295853991061, 0.00014263016055338085, 0.00014099254622124135, 0.00013957345800008625, 0.0001385276991641149, 0.00013720765127800405, 0.00013599597150459886, 0.00013494674931280315, 0.00013371968816500157, 0.00013283846783451736, 0.00013132279855199158, 0.0001296781119890511, 0.0001283805468119681, 0.00012712554598692805, 0.0001258437114302069, 0.00012473712558858097, 0.00012391561176627874, 0.0001233758230227977, 0.0001229580957442522, 0.00012222325312905014 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0003110146244580392, "first_to_last_ratio": 2.2098355110548322, "imbalance_abs_log_ratio": 0.7929180833727034, "last_quartile_mean": 0.00014074107457417995, "mean": 0.00020577548139044666, "normalized": [ 1.8085529821987474, 1.7406694730308092, 1.619926374791616, 1.5420513399817848, 1.4477144166533082, 1.3679714513872108, 1.309931951281633, 1.254598241859695, 1.1926238819658395, 1.119002359728262, 1.082854716042971, 1.0468411526236405, 0.9850613364068194, 0.9618428208148979, 0.9182052809224994, 0.877960480924469, 0.8608981306240351, 0.8408442911829569, 0.8148282643879068, 0.7879356949258838, 0.7604349554811285, 0.7438321978222967, 0.727404890131739, 0.7163770004088065, 0.7034258791916428, 0.6971552930824737, 0.6907054382632783, 0.684504508832364, 0.6799757639842041, 0.6759922505973077, 0.6720742215597147, 0.6678029589100581 ], "peak_layer": 1, "peak_normalized": 1.8085529821987474, "population_cv": 0.3383164219384334, "rest_mean": 0.00021605301460820355, "spike_contrast": 0.695555219569954, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001502768020145595, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00037215586053207517, 0.0003581870987545699, 0.00033334112958982587, 0.0003173163568135351, 0.00029790413100272417, 0.0002814949839375913, 0.00026955187786370516, 0.0002581655571702868, 0.00024541275342926383, 0.00023026324925012887, 0.00022282495046965778, 0.00021541424212045968, 0.00020270147069822997, 0.00019792366947513074, 0.00018894413369707763, 0.00018066274060402066, 0.00017715172725729644, 0.00017302513879258186, 0.00016767167835496366, 0.00016213784692808986, 0.0001564788690302521, 0.00015306242858059704, 0.00014968209143262357, 0.00014741282211616635, 0.00014474779891315848, 0.00014345746603794396, 0.00014213024405762553, 0.00014085424481891096, 0.00013992234016768634, 0.00013910263078287244, 0.00013829639647156, 0.00013741747534368187 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0021227982360869646, 0.00413061399012804, 0.005416445434093475, 0.006388718727976084, 0.0023577234242111444, 0.004388594534248114, 0.00574506726115942, 0.007066869642585516, 0.0029547461308538914, 0.004991721361875534, 0.006190058775246143, 0.0074937604367733, 0.00348988501355052, 0.006057556718587875, 0.0074848332442343235, 0.00897275935858488, 0.004072706215083599, 0.0063449000008404255, 0.007735534105449915, 0.00946879480034113, 0.004824656527489424, 0.007074959110468626, 0.009095527231693268, 0.010346688330173492, 0.004843301139771938, 0.007334613241255283, 0.009146629832684994, 0.010587968863546848, 0.0054801348596811295, 0.0077637480571866035, 0.009536821395158768, 0.011367839761078358 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015727686513855588, "first_to_last_ratio": 3.444355141493678, "imbalance_abs_log_ratio": 1.2367367001496994, "last_quartile_mean": 4.5662209231522866e-05, "mean": 8.846079299473786e-05, "normalized": [ 2.4938513501811554, 2.232208910931446, 1.9412909982664082, 1.7875424925486438, 1.6051944492471113, 1.4809426711898561, 1.3782256757419, 1.304162421055249, 1.1983748443875557, 1.116540877408076, 1.0705280549740306, 1.028906183424365, 0.9692497056236429, 0.9297086109864628, 0.876031508599092, 0.840542245421429, 0.8215073067800788, 0.7958492991481336, 0.7578128978794811, 0.7149268338582802, 0.6789684541053794, 0.6481555926886391, 0.6127683614407704, 0.5872234895762031, 0.5712935129257742, 0.5525148673059793, 0.5351006365732414, 0.5196040797164857, 0.49974182292452146, 0.4940518464188538, 0.4845470025727643, 0.4726329960989905 ], "peak_layer": 1, "peak_normalized": 2.4938513501811554, "population_cv": 0.525487396006855, "rest_mean": 9.469102304946218e-05, "spike_contrast": 0.5789096889426617, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.481755069922656e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002206080680480227, 0.00019746297039091587, 0.00017172814114019275, 0.00015812742640264332, 0.00014199677389115095, 0.0001310053630732, 0.00012191893620183691, 0.00011536724196048453, 0.00010600918903946877, 9.877009142655879e-05, 9.469976066611707e-05, 9.101785690290853e-05, 8.574059756938368e-05, 8.224276098189875e-05, 7.74944419390522e-05, 7.435503357555717e-05, 7.267118780873716e-05, 7.040146010695025e-05, 6.70367298880592e-05, 6.324299465632066e-05, 6.0062087868573144e-05, 5.733635771321133e-05, 5.42059751751367e-05, 5.1946255553048104e-05, 5.0537077186163515e-05, 4.88759033032693e-05, 4.733542664325796e-05, 4.596458893502131e-05, 4.4207557948539034e-05, 4.3704218114726245e-05, 4.2863412090810016e-05, 4.1809489630395547e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0002221399099653354, "first_to_last_ratio": 1.7678964788399845, "imbalance_abs_log_ratio": 0.5697904098088594, "last_quartile_mean": 0.00012565210272441618, "mean": 0.00016188397967198398, "normalized": [ 1.7325277794978797, 1.5951376332003655, 1.4520548687114199, 1.3729981481664384, 1.2857750985910605, 1.2257666436669268, 1.1762283345939986, 1.1372455678364286, 1.0889526987749056, 1.0509403055231545, 1.0259172538151122, 1.0049179037159137, 0.978085345877984, 0.9585059142501069, 0.9364886382982085, 0.9194669656082258, 0.9066971379228966, 0.893622399915035, 0.8769613699768213, 0.8605074489369314, 0.8461051105300481, 0.8336447776090111, 0.8211750960224249, 0.8107883690330443, 0.8025267555268564, 0.7926031470462279, 0.7851370690218832, 0.7775583575558719, 0.7689758331264998, 0.7655660871494063, 0.7607136802742225, 0.75640826022469 ], "peak_layer": 1, "peak_normalized": 1.7325277794978797, "population_cv": 0.2510639592948361, "rest_mean": 0.00016719473286583606, "spike_contrast": 0.796711177092359, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013320591242518275, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002804684918373823, 0.0002582272281870246, 0.00023506442084908485, 0.00022226640430744737, 0.00020814638992305845, 0.00019843198242597282, 0.00019041252380702645, 0.00018410183838568628, 0.00017628399655222893, 0.00017013039905577898, 0.00016607956786174327, 0.00016268010949715972, 0.00015833634824957699, 0.00015516675193794072, 0.00015160250768531114, 0.00014884697156958282, 0.00014677974104415625, 0.00014466315042227507, 0.00014196599659044296, 0.000139302370371297, 0.00013697086251340806, 0.00013495373423211277, 0.00013293509255163372, 0.00013125364785082638, 0.00012991622497793287, 0.00012830975174438208, 0.00012710111332125962, 0.000125874241348356, 0.00012448486813809723, 0.00012393288488965482, 0.00012314735795371234, 0.00012245037942193449 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19699589163064957, "first_to_last_ratio": 1.7678964443273588, "imbalance_abs_log_ratio": 0.5697903902870003, "last_quartile_mean": 0.11142954230308533, "mean": 0.1435603315476328, "normalized": [ 1.7325277292733527, 1.595137596567507, 1.4520549137362593, 1.3729980837787108, 1.285775112498371, 1.2257666266551688, 1.1762283724960378, 1.1372455675264566, 1.0889526722354506, 1.0509403711345948, 1.0259172522938773, 1.00491793591038, 0.9780853203912144, 0.9585059497750074, 0.9364885904027979, 0.9194669905174069, 0.9066971317518798, 0.8936224205946474, 0.8769613084029552, 0.8605074273314671, 0.84610509802098, 0.833644762521393, 0.821175085273337, 0.8107884103394608, 0.802526775587875, 0.7926031918850489, 0.7851370589142022, 0.7775583578743038, 0.7689758340628073, 0.7655660958716172, 0.7607136803251955, 0.7564082760502372 ], "peak_layer": 1, "peak_normalized": 1.7325277292733527, "population_cv": 0.2510639513231401, "rest_mean": 0.14826996044980156, "spike_contrast": 0.7967111822081809, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11812833547592164, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24872225522994995, 0.22899848222732544, 0.20845748484134674, 0.19710806012153625, 0.184586301445961, 0.17597146332263947, 0.16885973513126373, 0.16326335072517395, 0.156330406665802, 0.1508733481168747, 0.14728102087974548, 0.14426635205745697, 0.1404142528772354, 0.13760343194007874, 0.13444261252880096, 0.13199898600578308, 0.13016574084758759, 0.12828873097896576, 0.1258968561887741, 0.12353473156690598, 0.12146712839603424, 0.1196783185005188, 0.11788816750049591, 0.11639705300331116, 0.11521100997924805, 0.11378637701272964, 0.11271453648805618, 0.111626535654068, 0.1103944256901741, 0.10990492254495621, 0.10920830816030502, 0.10859022289514542 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.813338586653117e-05, "first_to_last_ratio": 2.039661655563216, "imbalance_abs_log_ratio": 0.7127839389845736, "last_quartile_mean": 3.3404258829250466e-05, "mean": 4.817700425974181e-05, "normalized": [ 1.6677237328958603, 1.5880861584886787, 1.4689677920146058, 1.4140322553861728, 1.3674596064698026, 1.3072824267629832, 1.2642946708581568, 1.2359968662296865, 1.1645552208108763, 1.101791124796405, 1.0797798333773554, 1.0672664620429624, 1.012803710836977, 0.9955853656459409, 0.9623936542224847, 0.9380946280712082, 0.9310631059210642, 0.9082494158597965, 0.8812030087749774, 0.8644086677601183, 0.8497891698936241, 0.8309347651333899, 0.7889921814106587, 0.7623244708429415, 0.7584228017816982, 0.7445258855646851, 0.7180452956289476, 0.7000407870401953, 0.6663749309645193, 0.6665299586713209, 0.6558653663654831, 0.637116679476424 ], "peak_layer": 1, "peak_normalized": 1.6677237328958603, "population_cv": 0.28258618188430573, "rest_mean": 4.9978354245131074e-05, "spike_contrast": 0.7693273401931907, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.844971433863975e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.034593338379636e-05, 7.650923362234607e-05, 7.077046757331118e-05, 6.812383799115196e-05, 6.588010728592053e-05, 6.298095104284585e-05, 6.090992974350229e-05, 5.9546626289375126e-05, 5.610478183371015e-05, 5.3080995712662116e-05, 5.2020557632204145e-05, 5.141770088812336e-05, 4.879384869127534e-05, 4.79643204016611e-05, 4.636524317902513e-05, 4.519458889262751e-05, 4.485583122004755e-05, 4.375673597678542e-05, 4.245372110744938e-05, 4.164462006883696e-05, 4.094029645784758e-05, 4.0031947719398886e-05, 3.8011279684724286e-05, 3.672650927910581e-05, 3.653853855212219e-05, 3.5869026760337874e-05, 3.4593271266203374e-05, 3.3725867979228497e-05, 3.21039478876628e-05, 3.211141665815376e-05, 3.1597628549207e-05, 3.0694372981088236e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018498812278266996, "first_to_last_ratio": 1.477115588497286, "imbalance_abs_log_ratio": 0.3900912594562188, "last_quartile_mean": 0.00012523605073511135, "mean": 0.00015057817927299766, "normalized": [ 1.364889512785021, 1.3222437098497068, 1.2702392636002102, 1.235819093505974, 1.2002761568067795, 1.1699209662016437, 1.143160310272995, 1.1216013166366416, 1.0931712967611817, 1.0676161315047876, 1.0503356923443616, 1.0352076251778792, 1.014929888181958, 1.0014649033450271, 0.9840777733314531, 0.96959004456439, 0.958410698647572, 0.9467186761221033, 0.9322053376846741, 0.9180751779000818, 0.904837007766356, 0.8925970349353171, 0.879899470446981, 0.8691033998232655, 0.8603564893899045, 0.8496764837250937, 0.8415817018035472, 0.8332582690137502, 0.8239257185712614, 0.820107268399535, 0.8147646079598274, 0.8099389729407196 ], "peak_layer": 1, "peak_normalized": 1.364889512785021, "population_cv": 0.1566386619724503, "rest_mean": 0.000153886474436149, "spike_contrast": 0.8624109810706347, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013271338539198042, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002055225777439773, 0.00019910105038434267, 0.000191270315553993, 0.00018608738901093602, 0.0001807353983167559, 0.00017616456898394972, 0.00017213499813806266, 0.00016888868412934244, 0.00016460774349980056, 0.00016075969324447215, 0.0001581576361786574, 0.00015587967936880887, 0.00015282629465218633, 0.00015079876175150275, 0.0001481806393712759, 0.00014599910355173051, 0.00014431573799811304, 0.00014255517453420907, 0.00014036978245712817, 0.00013824208872392774, 0.00013624870916828513, 0.00013440563634503633, 0.0001324936602031812, 0.00013086800754535943, 0.00012955091369803995, 0.00012794273789040744, 0.00012672384036704898, 0.0001254705130122602, 0.0001240652345586568, 0.00012349025928415358, 0.00012268577120266855, 0.00012195913586765528 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015959453048708383, "first_to_last_ratio": 1.2728956423420892, "imbalance_abs_log_ratio": 0.24129433847892473, "last_quartile_mean": 0.00012537911607068963, "mean": 0.00014144417764327955, "normalized": [ 1.1740787416792278, 1.1612764386268224, 1.1471117856731012, 1.1325725430810556, 1.1221203501929577, 1.1091452070987, 1.0952451640185767, 1.0850231158766837, 1.0710066115302646, 1.0558964823952193, 1.0428503514266292, 1.0346153464801762, 1.0229188081717178, 1.0151986196356395, 1.003700332970425, 0.9946300373693288, 0.9863513080075261, 0.9777764871930738, 0.967654233599155, 0.9595434059922948, 0.9513191006674802, 0.9429489101168639, 0.9334504238882736, 0.9221966853060849, 0.914156536930625, 0.9038869577772225, 0.8952163547729103, 0.8871546013922188, 0.8776639341179028, 0.8750860454698683, 0.8708633986378481, 0.8673416799041263 ], "peak_layer": 1, "peak_normalized": 1.1740787416792278, "population_cv": 0.09275162400716915, "rest_mean": 0.00014320399646681767, "spike_contrast": 0.9213510743517984, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001319411559961736, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001660666021052748, 0.00016425579087808728, 0.00016225228318944573, 0.0001601957919774577, 0.00015871739014983177, 0.0001568821317050606, 0.0001549160515423864, 0.00015347020234912634, 0.00015148764941841364, 0.0001493504096288234, 0.00014750511036254466, 0.00014634031686000526, 0.00014468590961769223, 0.00014359393389895558, 0.00014196756819728762, 0.0001406846276950091, 0.00013951364962849766, 0.00013830079114995897, 0.0001368690573144704, 0.00013572182797361165, 0.000134558547870256, 0.00013337463315110654, 0.00013203112757764757, 0.00013043935177847743, 0.00012930211960338056, 0.00012784954742528498, 0.00012662314111366868, 0.00012548285303637385, 0.00012414045340847224, 0.00012377582606859505, 0.00012317855725996196, 0.00012268043064977974 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031298474641516805, "first_to_last_ratio": 2.209030513243648, "imbalance_abs_log_ratio": 0.7925537374904509, "last_quartile_mean": 0.00014168421148497146, "mean": 0.0002071673161481158, "normalized": [ 1.8097158340785349, 1.7497804411308835, 1.611244862997228, 1.5435020349400896, 1.4351553463811126, 1.3679605489917586, 1.3130823229189843, 1.255818245825708, 1.1871164694634437, 1.122716192977336, 1.0885067728511792, 1.0444622322416663, 0.990152969051365, 0.9605806666596922, 0.9222549756846732, 0.8800158737132093, 0.8615285136033911, 0.8400222785845712, 0.8115627552032852, 0.7845983398409414, 0.7619432914958448, 0.7435905859348785, 0.7280762324918081, 0.7153162915808863, 0.7064441934578637, 0.6963930075638303, 0.6905250334602637, 0.686293775915277, 0.6782364888338203, 0.6751963304984743, 0.6708841537879804, 0.6673229378400193 ], "peak_layer": 1, "peak_normalized": 1.8097158340785349, "population_cv": 0.3382701486201724, "rest_mean": 0.00021748447411314205, "spike_contrast": 0.6963929896815652, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015145466313697397, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003749139723367989, 0.00036249731783755124, 0.00033379727392457426, 0.0003197631740476936, 0.0002973172813653946, 0.00028339671553112566, 0.0002720277407206595, 0.0002601644955575466, 0.0002459317329339683, 0.00023259010049514472, 0.0002255030267406255, 0.00021637843747157604, 0.00020512733317445964, 0.00019900091865565628, 0.00019106108811683953, 0.00018231052672490478, 0.00017848054994829, 0.00017402516095899045, 0.00016812927788123488, 0.0001625431323191151, 0.00015784974675625563, 0.00015404766600113362, 0.00015083359903655946, 0.00014819015632383525, 0.00014635214756708592, 0.00014426987036131322, 0.00014305421791505069, 0.00014217763964552432, 0.00014050843310542405, 0.0001398786116624251, 0.00013898526958655566, 0.0001382475020363927 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034565660171210766, 0.004928664769977331, 0.0060537648387253284, 0.006960673723369837, 0.0036145171616226435, 0.0051584807224571705, 0.006370538379997015, 0.007546419743448496, 0.00395671371370554, 0.005665866658091545, 0.006755253300070763, 0.007946331985294819, 0.004383862018585205, 0.006652849726378918, 0.007932198233902454, 0.009347881190478802, 0.004880861379206181, 0.006906712893396616, 0.008132148534059525, 0.009816600009799004, 0.005453306715935469, 0.007622495759278536, 0.009448722936213017, 0.010670131072402, 0.0055701653473079205, 0.007738742511719465, 0.009509840048849583, 0.010865328833460808, 0.005987230688333511, 0.00815181527286768, 0.009874933399260044, 0.011666664853692055 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015672678182454547, "first_to_last_ratio": 3.4451487067191406, "imbalance_abs_log_ratio": 1.236967069490208, "last_quartile_mean": 4.549202230919036e-05, "mean": 8.812991916329338e-05, "normalized": [ 2.495342024881723, 2.231858330120814, 1.9399329049931513, 1.7866550124027707, 1.6079479837593746, 1.479673880937976, 1.3811985381554308, 1.3042766606266647, 1.1954508396552082, 1.1168862767589318, 1.0692251193363627, 1.0303878333212484, 0.9676417435164101, 0.9260651349655372, 0.8738635380560305, 0.8395483633300559, 0.8220975248023003, 0.7970985234813915, 0.759112342831592, 0.7162079306537569, 0.6791607399033079, 0.6489075313744581, 0.615087961024076, 0.5868315353418163, 0.5699679008989185, 0.5521854623420828, 0.5335941062986479, 0.5198929115704046, 0.5024718770039331, 0.49422901772262035, 0.4850259920431499, 0.47217445788985424 ], "peak_layer": 1, "peak_normalized": 2.495342024881723, "population_cv": 0.5254832958744828, "rest_mean": 9.433179891655325e-05, "spike_contrast": 0.5792295824234721, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.4639768495690075e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021991429093759507, 0.0001966934942174703, 0.00017096613009925932, 0.0001574577618157491, 0.00014170832582749426, 0.00013040353951510042, 0.0001217249155160971, 0.00011494579666759819, 0.00010535498586250469, 9.843109728535637e-05, 9.423072333447635e-05, 9.080799645744264e-05, 8.527818863512948e-05, 8.161404548445717e-05, 7.701352296862751e-05, 7.398932939395308e-05, 7.245138840517029e-05, 7.024822843959555e-05, 6.690050940960646e-05, 6.311934703262523e-05, 5.985438110656105e-05, 5.7188168284483254e-05, 5.420765228336677e-05, 5.1717415772145614e-05, 5.02312250318937e-05, 4.866406015935354e-05, 4.702560545410961e-05, 4.581812027026899e-05, 4.428280590218492e-05, 4.3556363380048424e-05, 4.2745301470858976e-05, 4.161269680480473e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00022039370378479362, "first_to_last_ratio": 1.7624457042066763, "imbalance_abs_log_ratio": 0.5667024491058967, "last_quartile_mean": 0.00012504992537287762, "mean": 0.0001607882525149762, "normalized": [ 1.7301666786506156, 1.592657504255016, 1.4498087081325215, 1.3715848200709229, 1.2853506396348091, 1.224257634451794, 1.175979576607571, 1.1358564358860717, 1.0864628307036615, 1.050035766385231, 1.0256357250890054, 1.0049217142965674, 0.9775345112735686, 0.9578189361646354, 0.9358080993289649, 0.9184597363064851, 0.9072850761620224, 0.8934833681037547, 0.8779253472396409, 0.8613572157279928, 0.8467159963027754, 0.8352601414820391, 0.8224708066612338, 0.81131886291965, 0.8015955184587156, 0.7940537632065271, 0.7857694259399116, 0.7781749070905981, 0.7724085609032495, 0.7676732316943716, 0.7637136993200662, 0.75845476155001 ], "peak_layer": 1, "peak_normalized": 1.7301666786506156, "population_cv": 0.2498932502086342, "rest_mean": 0.0001660444721454513, "spike_contrast": 0.7974048446155259, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001324046665104106, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027819047681987286, 0.0002560806169640273, 0.0002331122086616233, 0.00022053472639527172, 0.00020666928321588784, 0.00019684624567162246, 0.0001890837011160329, 0.00018263237143401057, 0.00017469045997131616, 0.0001688334159553051, 0.0001649101759539917, 0.00016157960635609925, 0.00015717606584075838, 0.00015400603297166526, 0.00015046694898046553, 0.00014767753600608557, 0.00014588078192900866, 0.00014366162940859795, 0.00014116008242126554, 0.00013849612150806934, 0.0001361419854220003, 0.00013430001854430884, 0.00013224364374764264, 0.00013045054220128804, 0.00012888714263681322, 0.00012767451698891819, 0.0001263424928765744, 0.00012512138346210122, 0.00012419422273524106, 0.00012343283742666245, 0.0001227961911354214, 0.00012195061572128907 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19544734060764313, "first_to_last_ratio": 1.7624457434404193, "imbalance_abs_log_ratio": 0.5667024713668619, "last_quartile_mean": 0.11089552193880081, "mean": 0.14258862822316587, "normalized": [ 1.7301667593069614, 1.5926575752547714, 1.4498087309457477, 1.3715847721481056, 1.2853506436856033, 1.2242575941276408, 1.1759796246815233, 1.1358564592049036, 1.0864627991256137, 1.0500357507634608, 1.0256357136610779, 1.0049216567312906, 0.9775345659226533, 0.957818945514909, 0.935808091557486, 0.9184597083380731, 0.9072850362319335, 0.8934833286128027, 0.8779253174360568, 0.8613572174793438, 0.8467160247668938, 0.8352601831545204, 0.8224708111632975, 0.8113188687979516, 0.8015955037045681, 0.7940537757091718, 0.7857693860910103, 0.7781748832951502, 0.7724085309387246, 0.7676732205256189, 0.7637136996793319, 0.7584548214438025 ], "peak_layer": 1, "peak_normalized": 1.7301667593069614, "population_cv": 0.24989326289316924, "rest_mean": 0.14724989594132812, "spike_contrast": 0.797404859232464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11741778254508972, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.24670210480690002, 0.2270948588848114, 0.20672623813152313, 0.1955723911523819, 0.18327638506889343, 0.17456521093845367, 0.16768132150173187, 0.16196021437644958, 0.15491724014282227, 0.14972315728664398, 0.14624398946762085, 0.14329040050506592, 0.13938531279563904, 0.13657408952713013, 0.13343559205532074, 0.13096190989017487, 0.12936852872371674, 0.12740056216716766, 0.1251821666955948, 0.12281974405050278, 0.12073207646608353, 0.11909860372543335, 0.11727498471736908, 0.11568484455347061, 0.11429840326309204, 0.11322303861379623, 0.11204177886247635, 0.11095888912677765, 0.11013667285442352, 0.1094614714384079, 0.1088968887925148, 0.108147032558918 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.812703486502869e-05, "first_to_last_ratio": 2.0429306771418236, "imbalance_abs_log_ratio": 0.7143853833664048, "last_quartile_mean": 3.334769780849456e-05, "mean": 4.813682039639389e-05, "normalized": [ 1.670694398734861, 1.5890602892970849, 1.4709330435001327, 1.4135870318536654, 1.3688033811669116, 1.3072210435152478, 1.2659107055367187, 1.236022743120565, 1.164719165633699, 1.10067041168383, 1.075093368589445, 1.0685491843186352, 1.0112041551575939, 0.9945774787180313, 0.9602353052595208, 0.9352302198346567, 0.9313930096398116, 0.9092161974531191, 0.8814894006709005, 0.8653525310040872, 0.8506115464001769, 0.8320776390213925, 0.7932455071661803, 0.7619500965894176, 0.7566697664554205, 0.7419550822306056, 0.7141621566216388, 0.7010675908642553, 0.6716466869044494, 0.66592144202409, 0.6557448579818498, 0.6349845630520059 ], "peak_layer": 1, "peak_normalized": 1.670694398734861, "population_cv": 0.28301355753944435, "rest_mean": 4.992937398061191e-05, "spike_contrast": 0.7702285844110489, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.845703104161657e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.042191620916128e-05, 7.649230974493548e-05, 7.080603973008692e-05, 6.804558506701142e-05, 6.588984251720831e-05, 6.292546459008008e-05, 6.0936916270293295e-05, 5.9498204791452736e-05, 5.606587728834711e-05, 5.2982773922849447e-05, 5.1751576393144205e-05, 5.143656017025933e-05, 4.867615280090831e-05, 4.7875797463348135e-05, 4.622267442755401e-05, 4.501900912146084e-05, 4.483429802348837e-05, 4.3766776798292994e-05, 4.243209696142003e-05, 4.1655319364508614e-05, 4.094573523616418e-05, 4.005357186542824e-05, 3.8184316508704796e-05, 3.667785495053977e-05, 3.6423676647245884e-05, 3.571535853552632e-05, 3.437749546719715e-05, 3.374716470716521e-05, 3.233093593735248e-05, 3.2055340852821246e-05, 3.156547245453112e-05, 3.056613786611706e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00018337804431212135, "first_to_last_ratio": 1.471219679743085, "imbalance_abs_log_ratio": 0.38609177087911456, "last_quartile_mean": 0.00012464355040719965, "mean": 0.0001495256622092711, "normalized": [ 1.3620802882826015, 1.3194938018514704, 1.2677903290886852, 1.23356455122679, 1.198458119977671, 1.1684656874236388, 1.1415318441782234, 1.1198031889553155, 1.0910031274146845, 1.0660983196947673, 1.0498697336705507, 1.0350765279986567, 1.0145895852928877, 1.0013439705330982, 0.9836063325227735, 0.968549777523238, 0.9591067674489386, 0.9465820057822166, 0.9334010118478351, 0.9191551367815048, 0.9057362914957724, 0.8945632145340071, 0.8814992971854174, 0.8698868180749819, 0.8596244665411801, 0.8514889576556698, 0.8425072468470839, 0.8341767076391075, 0.8278204125037605, 0.8225904077486038, 0.8181934666462969, 0.8123426056325708 ], "peak_layer": 1, "peak_normalized": 1.3620802882826015, "population_cv": 0.1550560200772002, "rest_mean": 0.00015278581957789829, "spike_contrast": 0.8634362323882038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013192081241868435, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002036659570876509, 0.00019729818450286984, 0.0001895671884994954, 0.0001844495564000681, 0.00017920024401973933, 0.00017471560568083078, 0.00017068830493371934, 0.0001674393133725971, 0.0001631329650990665, 0.00015940905723255128, 0.0001569824671605602, 0.0001547705032862723, 0.00015170717961154878, 0.00014972662029322237, 0.00014707438822370023, 0.00014482304686680436, 0.00014341107453219593, 0.00014153830124996603, 0.00013956740440335125, 0.00013743728050030768, 0.00013543081877287477, 0.00013376015704125166, 0.0001318067661486566, 0.00013007040251977742, 0.00012853591761086136, 0.00012731945025734603, 0.00012597645400092006, 0.00012473082460928708, 0.00012378039536997676, 0.00012299837544560432, 0.0001223409199155867, 0.00012146606604801491 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00015817073654034175, "first_to_last_ratio": 1.2671327323753674, "imbalance_abs_log_ratio": 0.23675665700060805, "last_quartile_mean": 0.00012482570491556544, "mean": 0.00014045925900063594, "normalized": [ 1.1719218726461313, 1.1599555891510773, 1.143372783062998, 1.1305925991754215, 1.1202702784385838, 1.106568447670676, 1.0931409560371117, 1.082952696795682, 1.0683580206124534, 1.05266391584401, 1.0427272032106563, 1.034492263470324, 1.0219716035750168, 1.0154196846947314, 1.0039021031599646, 0.9939448772529704, 0.9863124890252286, 0.9768583563343864, 0.9682383265025308, 0.9606909958387225, 0.9520472410590707, 0.9456933063501335, 0.9351354973759991, 0.9231938712502704, 0.9140830768869257, 0.9051693342733999, 0.8960380266364146, 0.8881305740575848, 0.8822109405764296, 0.8783208779569746, 0.8754534745501711, 0.8701687165279489 ], "peak_layer": 1, "peak_normalized": 1.1719218726461313, "population_cv": 0.09095723041158388, "rest_mean": 0.00014217518737832843, "spike_contrast": 0.9227576779061385, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013119324576109647, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016460727783851326, 0.00016292650252580643, 0.00016059729387052357, 0.0001588021987117827, 0.0001573523331899196, 0.00015542778419330716, 0.00015354176866821945, 0.00015211073332466185, 0.00015006077592261136, 0.00014785639359615743, 0.0001464606903027743, 0.00014530401676893234, 0.00014354537415783852, 0.00014262509648688138, 0.0001410073455190286, 0.00013960876094643027, 0.00013853672135155648, 0.0001372088008793071, 0.00013599803787656128, 0.00013493794540409, 0.00013372385001275688, 0.00013283138105180115, 0.00013134843902662396, 0.0001296711270697415, 0.0001283914316445589, 0.00012713941396214068, 0.00012585683725774288, 0.00012474616232793778, 0.00012391469499561936, 0.00012336829968262464, 0.00012296554632484913, 0.00012222325312905014 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00031102553475648165, "first_to_last_ratio": 2.2098501973960105, "imbalance_abs_log_ratio": 0.7929247292485178, "last_quartile_mean": 0.00014074507635086775, "mean": 0.00020578417843353236, "normalized": [ 1.8085758305333837, 1.7408578335592815, 1.6197863489261306, 1.5416881776712252, 1.4477928223214513, 1.3679195769143782, 1.3100342829404608, 1.2546744848740619, 1.192398672058626, 1.1190508147725688, 1.082828610043334, 1.0467691899805913, 0.9849335746234156, 0.9618381641711736, 0.9182125099812896, 0.877817233347749, 0.8611427657722097, 0.8409392227987704, 0.8149468534272446, 0.7879793317746896, 0.7604794717340572, 0.7439065501103351, 0.7274095758071225, 0.7164574630456112, 0.7035124756224637, 0.6971408207181957, 0.690610058298894, 0.684463275349207, 0.6799680284055707, 0.6760380727842366, 0.6720531720343774, 0.6677747355978938 ], "peak_layer": 1, "peak_normalized": 1.8085758305333837, "population_cv": 0.33829873005197686, "rest_mean": 0.0002160599310796156, "spike_contrast": 0.6956177084463702, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015029511414468288, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003721762914210558, 0.00035824099904857576, 0.00033332640305161476, 0.00031725503504276276, 0.00029793285648338497, 0.0002814962062984705, 0.0002695843286346644, 0.00025819215807132423, 0.0002453767810948193, 0.00023028295254334807, 0.00022282899590209126, 0.00021540853776969016, 0.00020268374646548182, 0.00019793107639998198, 0.0001889536069938913, 0.00018064089817926288, 0.00017720955656841397, 0.0001730519870761782, 0.00016770316869951785, 0.00016215367941185832, 0.00015649464330635965, 0.00015308419824577868, 0.00014968938194215298, 0.00014743561041541398, 0.00014477173681370914, 0.00014346055104397237, 0.00014211662346497178, 0.00014085171278566122, 0.00013992666208650917, 0.00013911793939769268, 0.00013829790987074375, 0.00013741747534368187 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.028097106143832207, 0.014136734418570995, 0.014224153943359852, 0.014337832108139992, 0.014408928342163563, 0.009704872034490108, 0.009773381985723972, 0.00987160298973322, 0.00994801800698042, 0.007515520788729191, 0.007602147292345762, 0.007658426649868488, 0.007734050042927265, 0.0062688966281712055, 0.006336068734526634, 0.006399397272616625, 0.006462901830673218, 0.005432929843664169, 0.005504137370735407, 0.005560056772083044, 0.005631812382489443, 0.004873365629464388, 0.004937449004501104, 0.005005750339478254, 0.005022916477173567, 0.0044684382155537605, 0.004500201437622309, 0.004573327023535967, 0.00463611725717783, 0.004166214261204004, 0.0042496914975345135, 0.004283483140170574 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00014323742470878642, "first_to_last_ratio": 1.6208859606473658, "imbalance_abs_log_ratio": 0.48297288904334346, "last_quartile_mean": 8.83698348843609e-05, "mean": 0.00011961164659624046, "normalized": [ 0.879665217639624, 1.4482702231163533, 1.4522235545753037, 1.2017346580032677, 1.034069363337856, 1.36550769437152, 1.149442648944982, 1.0492523703464898, 1.0286339735981567, 1.1048881958426662, 1.2960670227766034, 1.1342833655748923, 0.9483258681651744, 1.1110988010941807, 1.0051223270547462, 0.9186295907129635, 0.9340104771989933, 1.0935042558126027, 0.9904421395235656, 1.070541841916413, 0.9831461473937306, 1.1165574278336847, 0.8906099662074067, 0.8835226812669187, 0.8406995671395937, 0.8651479324264472, 0.7705397043177317, 0.6783637574582257, 0.6658814129635636, 0.7912689330389562, 0.6809428819470401, 0.6176059984003462 ], "peak_layer": 3, "peak_normalized": 1.4522235545753037, "population_cv": 0.21063690333645962, "rest_mean": 0.00012087627045629133, "spike_contrast": 0.9330423359872595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011278267775196582, "top_five_layers": [ 3, 2, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010521820513531566, 0.00017322998610325158, 0.00017370285058859736, 0.00014374146121554077, 0.000123686739243567, 0.0001633306237636134, 0.00013748672790825367, 0.00012550280371215194, 0.00012303660332690924, 0.0001321574964094907, 0.00015502471069339663, 0.00013567350106313825, 0.00011343081860104576, 0.0001329003571299836, 0.00012022433656966314, 0.00010987879795720801, 0.0001117185311159119, 0.00013079584459774196, 0.00011846841516671702, 0.00012804927246179432, 0.00011759572953451425, 0.00013355327246244997, 0.00010652732453309, 0.00010567960271146148, 0.00010055745951831341, 0.00010348176874686033, 9.216552280122414e-05, 8.114020602079108e-05, 7.964717224240303e-05, 9.464497998123989e-05, 8.144869934767485e-05, 7.387287041638047e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00015447182067873655, "first_to_last_ratio": 1.6506746305835311, "imbalance_abs_log_ratio": 0.5011840713694677, "last_quartile_mean": 9.358102306578076e-05, "mean": 0.00012785212516064348, "normalized": [ 0.8854544129111614, 1.4719853585239924, 1.4439160513495162, 1.202741955390318, 1.0425045283644727, 1.3767992039076578, 1.1738833181051227, 1.0683704265103389, 1.0407410271769484, 1.136073438345882, 1.2855554885667506, 1.129314905977533, 0.953735223493217, 1.1222184469907663, 1.0164965849441758, 0.9298350247961144, 0.9320412792829462, 1.0837813144211186, 0.9857736034320764, 1.0482233302985868, 0.9656597421723547, 1.0923472818519362, 0.884471876187184, 0.8724974481708365, 0.826330173068434, 0.8536374096219558, 0.7648976975162065, 0.681485935203229, 0.6604246501739865, 0.7781176964145518, 0.6752223987471391, 0.6154627680834903 ], "peak_layer": 2, "peak_normalized": 1.4719853585239924, "population_cv": 0.21455917947613024, "rest_mean": 0.00012955063343893185, "spike_contrast": 0.9160910858365683, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011868018045788631, "top_five_layers": [ 2, 3, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001132072284235619, 0.00018819645629264414, 0.00018460773571860045, 0.00015377311501652002, 0.00013328641944099218, 0.00017602670413907617, 0.0001500834769103676, 0.00013659342948812991, 0.00013306095206644386, 0.00014524940343108028, 0.00016436100122518837, 0.00014438531070481986, 0.00012193707516416907, 0.0001434780133422464, 0.00012996124860364944, 0.00011888138396898285, 0.00011916345829376951, 0.00013856374425813556, 0.00012603325012605637, 0.00013401758042164147, 0.0001234616502188146, 0.00013965892139822245, 0.00011308160901535302, 0.00011155065294587985, 0.00010564806871116161, 0.00010913935693679377, 9.779379615793005e-05, 8.71294250828214e-05, 8.443669503321871e-05, 9.948400111170486e-05, 8.632861863588914e-05, 7.86882228567265e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.13698715530335903, "first_to_last_ratio": 1.6506746463483835, "imbalance_abs_log_ratio": 0.5011840809200188, "last_quartile_mean": 0.08298858627676964, "mean": 0.11338054295629263, "normalized": [ 0.8854544185536968, 1.4719853260144444, 1.4439159993116888, 1.2027420311824657, 1.042504519270847, 1.3767991933658148, 1.1738833593963587, 1.0683704218985235, 1.0407409783116481, 1.1360735196220915, 1.285555449073044, 1.129314933056767, 0.9537352653592515, 1.1222184500196979, 1.0164965450422543, 0.9298350370138166, 0.9320412867497836, 1.0837813135283418, 0.9857735942006465, 1.0482233283425098, 0.9656597514448471, 1.0923472716531482, 0.8844718771212735, 0.8724974491223123, 0.82633015871154, 0.8536374112441639, 0.7648976624892803, 0.681485927333974, 0.660424633911195, 0.7781176812354046, 0.6752224228414072, 0.6154627835777616 ], "peak_layer": 2, "peak_normalized": 1.4719853260144444, "population_cv": 0.2145591770090877, "rest_mean": 0.11488679713673061, "spike_contrast": 0.9160910827435619, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10524677038192749, "top_five_layers": [ 2, 3, 6, 11, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.10039330273866653, 0.16689449548721313, 0.16371197998523712, 0.1363675445318222, 0.11819972842931747, 0.1561022400856018, 0.13309553265571594, 0.12113241851329803, 0.11799977719783783, 0.1288086324930191, 0.14575697481632233, 0.1280423402786255, 0.10813502222299576, 0.1272377371788025, 0.11525093019008636, 0.10542520135641098, 0.1056753471493721, 0.12287971377372742, 0.11176754534244537, 0.11884813010692596, 0.10948702692985535, 0.12385092675685883, 0.10028190165758133, 0.09892423450946808, 0.09368976205587387, 0.09678587317466736, 0.08672451227903366, 0.07726724445819855, 0.07487930357456207, 0.0882234051823616, 0.07655708491802216, 0.06978150457143784 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.7160250005617854e-05, "first_to_last_ratio": 0.7388534944946782, "imbalance_abs_log_ratio": 0.30265562600345663, "last_quartile_mean": 6.382895981005277e-05, "mean": 5.960399772675373e-05, "normalized": [ 0.46158176249046556, 0.8375260596839308, 0.8768757910510345, 0.7241656982213429, 0.6710453705877814, 1.0089930699998053, 0.8842011269201543, 0.865421443073546, 0.883514351886623, 0.9622394496647411, 1.1256912972870226, 1.0210990974149952, 0.9232791878576234, 1.0980011773561495, 1.0212452171652862, 0.9544893655323485, 1.0133393696232265, 1.159814348379829, 0.9921993085906312, 1.1469178460749072, 1.0774269793999733, 1.3924548133021735, 1.1792402184404436, 1.1521666823514416, 1.1069955376913123, 1.1868921569761495, 1.1034570471301723, 0.9847635589252388, 0.9837749617675684, 1.1772881708824627, 1.0498861530819708, 0.9740133811896496 ], "peak_layer": 22, "peak_normalized": 1.3924548133021735, "population_cv": 0.17243044689052606, "rest_mean": 5.759890990264938e-05, "spike_contrast": 1.22279175241262, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.043147197691724e-05, "top_five_layers": [ 22, 26, 23, 30, 18 ], "top_five_spike_overlap": 2 }, "values": [ 2.751211832219269e-05, 4.991990135749802e-05, 5.226530265645124e-05, 4.316317063057795e-05, 3.9996986743062735e-05, 6.014002065057866e-05, 5.2701921958941966e-05, 5.158257772563957e-05, 5.266098742140457e-05, 5.735331797040999e-05, 6.709570152452216e-05, 6.0861588281113654e-05, 5.503113061422482e-05, 6.544525967910886e-05, 6.087029760237783e-05, 5.689138197340071e-05, 6.0399077483452857e-05, 6.912957178428769e-05, 5.9139045333722606e-05, 6.836088869022205e-05, 6.421895523089916e-05, 8.299587352667004e-05, 7.028743129922077e-05, 6.867374031571671e-05, 6.598135951207951e-05, 7.074351742630824e-05, 6.577045132871717e-05, 5.869584492756985e-05, 5.863692058483139e-05, 7.017108146101236e-05, 6.2577411881648e-05, 5.8055091358255595e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6127787956938846e-05, "first_to_last_ratio": 0.8333328618421979, "imbalance_abs_log_ratio": 0.18232212258347721, "last_quartile_mean": 6.735338365615462e-05, "mean": 6.689512088087213e-05, "normalized": [ 0.49022256384162527, 0.923222562655389, 0.9170289550360846, 0.7742286879065733, 0.7345865645537341, 1.0394905128518102, 0.9423822415780315, 0.8911705562121757, 0.9138181637864031, 1.0330628394205372, 1.1540375572503838, 1.0455389170050469, 0.9297420351318767, 1.1341544713967917, 1.0374336268991433, 0.9867016024211971, 1.0073449670971224, 1.1743962796261593, 1.0459325434797282, 1.1475080782429012, 1.066139536390961, 1.3297286391604237, 1.1245973641634917, 1.1027270030076322, 1.04892482254714, 1.125623033590473, 1.0476398534474196, 0.9361019469456514, 0.930671663614635, 1.0896207315037236, 0.9754527388499041, 0.9007689403858299 ], "peak_layer": 22, "peak_normalized": 1.3297286391604237, "population_cv": 0.14905429467389167, "rest_mean": 6.522988486621116e-05, "spike_contrast": 1.1633838617941636, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.58873953600414e-05, "top_five_layers": [ 22, 18, 11, 20, 14 ], "top_five_spike_overlap": 1 }, "values": [ 3.2793497666716576e-05, 6.17590849287808e-05, 6.134476279839873e-05, 5.179212166694924e-05, 4.9140257033286616e-05, 6.953684351174161e-05, 6.304077396634966e-05, 5.961496208328754e-05, 6.112997652962804e-05, 6.910686352057382e-05, 7.719948189333081e-05, 6.994145223870873e-05, 6.219520582817495e-05, 7.586940046167001e-05, 6.939924787729979e-05, 6.600552296731621e-05, 6.738646334270015e-05, 7.856138108763844e-05, 6.996778392931446e-05, 7.676269160583615e-05, 7.13195331627503e-05, 8.895235805539414e-05, 7.523007661802694e-05, 7.37670561647974e-05, 7.016795279923826e-05, 7.529868889832869e-05, 7.00819946359843e-05, 6.26206528977491e-05, 6.225739343790337e-05, 7.289031054824591e-05, 6.525302887894213e-05, 6.025704715284519e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.866022898866504e-05, "first_to_last_ratio": 0.40574519905788725, "imbalance_abs_log_ratio": 0.9020299049238899, "last_quartile_mean": 4.599001795213553e-05, "mean": 3.414976504245715e-05, "normalized": [ 0.2445526901503676, 0.5596630531980494, 0.5376964344799271, 0.48012169469058175, 0.4798804572179837, 0.6900344095639286, 0.7152325131545565, 0.6642055410913166, 0.7554442803991458, 0.9343372865664744, 0.9240467958883766, 0.8592444663674569, 0.8263415713413451, 1.0773994704320806, 0.959597390691018, 0.9894020700335048, 0.9760686528076964, 1.1936439731862338, 1.099336047654388, 1.1448507248259905, 1.0495120137852647, 1.4542665199347329, 1.3223844598378258, 1.2890134714336634, 1.2402412095084396, 1.4030995672846114, 1.3877316376933018, 1.246635573851661, 1.2962845787221184, 1.4847061188341568, 1.3923191445524272, 1.3227061808213745 ], "peak_layer": 30, "peak_normalized": 1.4847061188341568, "population_cv": 0.32662955778396485, "rest_mean": 3.2435424482510255e-05, "spike_contrast": 1.3382653305362568, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.340720406617038e-05, "top_five_layers": [ 30, 22, 26, 31, 27 ], "top_five_spike_overlap": 1 }, "values": [ 8.351416909135878e-06, 1.9112361769657582e-05, 1.8362206901656464e-05, 1.6396043065469712e-05, 1.6387804862461053e-05, 2.3564512957818806e-05, 2.4425022274954244e-05, 2.2682463168166578e-05, 2.5798244678298943e-05, 3.1907398806652054e-05, 3.1555980967823416e-05, 2.9342996640480123e-05, 2.8219370506121777e-05, 3.679293877212331e-05, 3.277002542745322e-05, 3.378784822416492e-05, 3.333251515869051e-05, 4.0762661228654906e-05, 3.7542067730100825e-05, 3.909638326149434e-05, 3.584058868000284e-05, 4.9662859964882955e-05, 4.515911859925836e-05, 4.4019507186021656e-05, 4.2353945900686085e-05, 4.791552055394277e-05, 4.7390709369210526e-05, 4.257231194060296e-05, 4.4267813791520894e-05, 5.070236511528492e-05, 4.754737165058032e-05, 4.517010529525578e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00021511870181711856, "first_to_last_ratio": 1.067203936185183, "imbalance_abs_log_ratio": 0.06504208450186474, "last_quartile_mean": 0.00020157225299044512, "mean": 0.00023711549874860793, "normalized": [ 0.5199215340336985, 0.9694171998286779, 0.9952311992308057, 0.8139231798698612, 0.8071491568326964, 1.1522101416966866, 1.0127543403881063, 0.9872470097646113, 0.9739878350012732, 1.0629045356723543, 1.2654986331851705, 1.158397891601516, 1.0251221075049282, 1.1837639471542862, 1.126357049236326, 1.0240452993420521, 1.0679809871494306, 1.2152134226291609, 1.0519957926786496, 1.2134701299884076, 1.1240302451392508, 1.3270417853241223, 1.0642149202553777, 1.0573092573282454, 0.9986311292560726, 1.01615500686031, 0.8954142860763721, 0.793725079326075, 0.7548043496765433, 0.8883375219775711, 0.764305251609244, 0.6894397743821165 ], "peak_layer": 22, "peak_normalized": 1.3270417853241223, "population_cv": 0.17546356070810173, "rest_mean": 0.00023209894856403548, "spike_contrast": 1.138328593816985, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002642048697452992, "top_five_layers": [ 22, 11, 18, 20, 14 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012328145385254174, 0.00022986384283285588, 0.0002359847421757877, 0.00019299380073789507, 0.00019138757488690317, 0.00027320688241161406, 0.00024013975053094327, 0.00023409156710840762, 0.00023094761127140373, 0.0002520311390981078, 0.00030006933957338333, 0.0002746740938164294, 0.00024307233979925513, 0.00028068877873010933, 0.0002670767134986818, 0.0002428170118946582, 0.00025323484442196786, 0.0002881459367927164, 0.00024944450706243515, 0.0002877325750887394, 0.0002665249921847135, 0.00031466217478737235, 0.0002523418515920639, 0.00025070441188290715, 0.0002367909182794392, 0.00024094610125757754, 0.00021231660502962768, 0.00018820451805368066, 0.00017897580983117223, 0.00021063859458081424, 0.00018122862093150616, 0.00016347685595974326 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014079716056585312, 0.014167090877890587, 0.014255538582801819, 0.014355218037962914, 0.009655896574258804, 0.009734091348946095, 0.009828771464526653, 0.009915035218000412, 0.007479202933609486, 0.007559895049780607, 0.0076358383521437645, 0.007706487085670233, 0.0062490664422512054, 0.006315518636256456, 0.0063744354993104935, 0.006439714692533016, 0.0054116579703986645, 0.005481995176523924, 0.005541813559830189, 0.005614723544567823, 0.004862982779741287, 0.004923288244754076, 0.0049925679340958595, 0.005004179198294878, 0.004448203835636377, 0.004490054678171873, 0.00455933902412653, 0.004625064320862293, 0.004160413984209299, 0.004242690280079842, 0.004270957782864571, 0.004303553607314825 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.6855086818832206e-05, "first_to_last_ratio": 1.1121706932103832, "imbalance_abs_log_ratio": 0.10631368513677814, "last_quartile_mean": 3.313797697046539e-05, "mean": 3.537216105087282e-05, "normalized": [ 1.3329674878497746, 1.065596392820356, 0.9206330800727252, 0.8842664165696186, 1.1462361065020723, 1.065158463293282, 0.99582829507555, 0.9247019781352203, 1.132583771771479, 1.005206548698436, 0.979948053013938, 0.967083333200755, 1.1057115808592233, 1.043785177992708, 0.995436955984306, 0.9279460300710988, 1.1098294348763655, 1.0288442525987258, 1.0234795644677463, 0.9449806453153974, 1.0109568163965834, 1.0260998393764513, 0.9416381672270038, 0.926379336673748, 0.9884731771330477, 0.9341156116917169, 0.9458688462068515, 0.9053302302997448, 0.9662542702797254, 0.9495526789453999, 0.9100381269887573, 0.8950693296121922 ], "peak_layer": 1, "peak_normalized": 1.3329674878497746, "population_cv": 0.09165679612351446, "rest_mean": 3.551162256001219e-05, "spike_contrast": 0.9748658722370791, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.461906890152022e-05, "top_five_layers": [ 1, 5, 9, 17, 13 ], "top_five_spike_overlap": 0 }, "values": [ 4.714994065579958e-05, 3.769244722207077e-05, 3.256478157709353e-05, 3.127841409877874e-05, 4.054484816151671e-05, 3.767695670831017e-05, 3.5224598832428455e-05, 3.270870729465969e-05, 4.006193557870574e-05, 3.555632792995311e-05, 3.4662880352698267e-05, 3.420782741159201e-05, 3.911140811396763e-05, 3.692093741847202e-05, 3.521075632306747e-05, 3.2823456422192976e-05, 3.9257065509445965e-05, 3.6392444599187e-05, 3.620268398663029e-05, 3.342600757605396e-05, 3.575972732505761e-05, 3.6295368772698566e-05, 3.330777690280229e-05, 3.276803909102455e-05, 3.49644324160181e-05, 3.3041687856893986e-05, 3.3457425161032006e-05, 3.202348671038635e-05, 3.417850166442804e-05, 3.358773028594442e-05, 3.2190015190280974e-05, 3.166053647873923e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.015105893515283e-05, "first_to_last_ratio": 0.6588274570259804, "imbalance_abs_log_ratio": 0.4172936042156455, "last_quartile_mean": 0.00010647865110513521, "mean": 9.056432463694364e-05, "normalized": [ 0.8598786252117567, 0.7452797309823858, 0.6694431442719111, 0.6417832925446045, 0.8822867975473095, 0.8318257491363829, 0.804532248804658, 0.7617644993607329, 0.969439786050785, 0.9144645011121496, 0.8951531228323205, 0.876468558935897, 1.0407786059783084, 1.0056397226462848, 0.9828357943952414, 0.9490839045924557, 1.1213662721801265, 1.0774120583919133, 1.0554308943167812, 1.0307906296953837, 1.1314055055223455, 1.1556700189426168, 1.1045075200854564, 1.0869669205578247, 1.2145839027446308, 1.1552275050200584, 1.1514380979421175, 1.1280886596086177, 1.2254687972249847, 1.20187962367999, 1.1639632986601018, 1.165142211023867 ], "peak_layer": 29, "peak_normalized": 1.2254687972249847, "population_cv": 0.1618495930711073, "rest_mean": 8.82393912804589e-05, "spike_contrast": 1.1686273359956587, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010311896476196125, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 7.787432696204633e-05, 6.74957555020228e-05, 6.062766624381766e-05, 5.812267045257613e-05, 7.990370795596391e-05, 7.533373718615621e-05, 7.286191976163536e-05, 6.898868741700426e-05, 8.779665949987248e-05, 8.281785994768143e-05, 8.106893801596016e-05, 7.937678310554475e-05, 9.425741154700518e-05, 9.107508230954409e-05, 8.900985994841903e-05, 8.595314284320921e-05, 0.00010155577911064029, 9.757509542396292e-05, 9.558438614476472e-05, 9.335285722045228e-05, 0.00010246497549815103, 0.00010466247476870194, 0.00010002897761296481, 9.844042506301776e-05, 0.00010999797086697072, 0.00010462239879416302, 0.00010427921370137483, 0.00010216458758804947, 0.00011098375398432836, 0.00010884741641348228, 0.00010541355004534125, 0.00010552031744737178 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06221066089347005, "first_to_last_ratio": 0.6588274447143223, "imbalance_abs_log_ratio": 0.4172936229028725, "last_quartile_mean": 0.09442633483558893, "mean": 0.0803133494919166, "normalized": [ 0.8598786809861813, 0.7452797356777653, 0.6694431594543391, 0.6417832788211475, 0.8822868217673959, 0.83182573854958, 0.804532202278358, 0.7617644434307961, 0.9694397651274825, 0.9144645475330049, 0.8951531251987148, 0.8764685422804359, 1.0407786734529123, 1.0056396718254574, 0.9828357789429103, 0.9490838578168909, 1.121366176654653, 1.0774119964231388, 1.0554309636293913, 1.030790617723408, 1.131405440813584, 1.1556700726993148, 1.1045075638748008, 1.0869669141870728, 1.214583886452764, 1.155227565075462, 1.151438141297865, 1.1280886745503538, 1.2254688318483915, 1.20187965027907, 1.1639633335063522, 1.1651421478410058 ], "peak_layer": 29, "peak_normalized": 1.2254688318483915, "population_cv": 0.16184959877781283, "rest_mean": 0.07825157529226055, "spike_contrast": 1.1686273385361958, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0914469301700592, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.06905973702669144, 0.05985591188073158, 0.05376522243022919, 0.051543764770030975, 0.0708594098687172, 0.06680671125650406, 0.0646146759390831, 0.06117985397577286, 0.07785895466804504, 0.0734437108039856, 0.07189274579286575, 0.07039212435483932, 0.08358842134475708, 0.08076629042625427, 0.07893483340740204, 0.07622410356998444, 0.09006067365407944, 0.08653056621551514, 0.08476519584655762, 0.08278624713420868, 0.09086696058511734, 0.09281573444604874, 0.08870670199394226, 0.0872979536652565, 0.09754730015993118, 0.09278019517660141, 0.09247585386037827, 0.09060057997703552, 0.09842150658369064, 0.09652698040008545, 0.09348179399967194, 0.09357646852731705 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.077007694533677e-06, "first_to_last_ratio": 0.4632676420751026, "imbalance_abs_log_ratio": 0.7694503312712233, "last_quartile_mean": 1.5276283193088602e-05, "mean": 1.1745167029175718e-05, "normalized": [ 0.5860812018784138, 0.5338863181232825, 0.4896831424227033, 0.48432444022666415, 0.7227742595277804, 0.658356157633798, 0.6679626701195096, 0.6773027265164295, 0.9317627933811012, 0.8396840560424388, 0.8002283441953012, 0.8237560838623014, 0.9947731150675175, 0.9485376018841178, 1.012754293639493, 0.980582798985168, 1.1861357451339187, 1.074396404738039, 1.1583203932505854, 1.1590437197037688, 1.1716087388237089, 1.3380122311407743, 1.182136735603702, 1.17274255168395, 1.3256589211015541, 1.1785808128912905, 1.3283612705891032, 1.2366027296837898, 1.3388375403515687, 1.3849270873253279, 1.3156503626951777, 1.2965347517777213 ], "peak_layer": 30, "peak_normalized": 1.3849270873253279, "population_cv": 0.2782190894097951, "rest_mean": 1.1227440423950881e-05, "spike_contrast": 1.2951207174852213, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4540890697389841e-05, "top_five_layers": [ 30, 29, 22, 27, 25 ], "top_five_spike_overlap": 2 }, "values": [ 6.883621608722024e-06, 6.2705839809495956e-06, 5.751410299126292e-06, 5.6884714467742015e-06, 8.48910440254258e-06, 7.732503036095295e-06, 7.845333129807841e-06, 7.955033652251586e-06, 1.0943709639832377e-05, 9.862229489954188e-06, 9.39881556405453e-06, 9.67515279626241e-06, 1.168377639260143e-05, 1.1140732567582745e-05, 1.1894968338310719e-05, 1.1517108760017436e-05, 1.3931362445873674e-05, 1.2618965229194146e-05, 1.3604666492028628e-05, 1.3613162082037888e-05, 1.3760740330326371e-05, 1.5715177141828462e-05, 1.3884393410990015e-05, 1.377405715174973e-05, 1.5570085452054627e-05, 1.38426285047899e-05, 1.56018249981571e-05, 1.4524105608870741e-05, 1.572487053635996e-05, 1.62661999638658e-05, 1.5452533261850476e-05, 1.5228017218760215e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.028231200616574e-05, "first_to_last_ratio": 0.5725175422355306, "imbalance_abs_log_ratio": 0.5577119025637401, "last_quartile_mean": 0.00010529338851483772, "mean": 8.693937525094952e-05, "normalized": [ 0.6520176082954547, 0.61792639701702, 0.5860559002851884, 0.5711155588169554, 0.8204737244004536, 0.784378974164202, 0.772878878140371, 0.7422194596148288, 0.9603819210570083, 0.917062379210463, 0.9022792075902993, 0.8821977885039085, 1.057360487646653, 1.0255127545988867, 1.0067830966339102, 0.9749202156934387, 1.1510091959216098, 1.1091409226242501, 1.0872456061539109, 1.0647793564016073, 1.1683829082644246, 1.1939706281941995, 1.1404893610428462, 1.1225154239128359, 1.2535795945499137, 1.1915479696415308, 1.1872717446378291, 1.1619194447200984, 1.2611193116864508, 1.2370940006457103, 1.1981927894482458, 1.1981773904854955 ], "peak_layer": 29, "peak_normalized": 1.2611193116864508, "population_cv": 0.20676778332368514, "rest_mean": 8.410921549916061e-05, "spike_contrast": 1.215351222858925, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001022222379106097, "top_five_layers": [ 29, 25, 30, 31, 32 ], "top_five_spike_overlap": 1 }, "values": [ 5.6686003517825156e-05, 5.3722134907729924e-05, 5.095133383292705e-05, 4.965242987964302e-05, 7.133147300919518e-05, 6.81934179738164e-05, 6.719360681017861e-05, 6.452809611801058e-05, 8.349500421900302e-05, 7.972883031470701e-05, 7.844359060982242e-05, 7.669772458029911e-05, 9.192626021103933e-05, 8.915743819670752e-05, 8.75290934345685e-05, 8.475895447190851e-05, 0.00010006802040152252, 9.642801887821406e-05, 9.452445374336094e-05, 9.257125202566385e-05, 0.00010157848009839654, 0.00010380306048318744, 9.915343252941966e-05, 9.759078966453671e-05, 0.00010898542677750811, 0.00010359243606217206, 0.00010322066373191774, 0.00010101655061589554, 0.00010964092507492751, 0.0001075521795428358, 0.00010417013254482299, 0.00010416879376862198 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 5.289650744089158e-05, "first_to_last_ratio": 0.5036607689093815, "imbalance_abs_log_ratio": 0.6858523150899144, "last_quartile_mean": 0.00010502407712920103, "mean": 8.374395429200376e-05, "normalized": [ 0.5462883602941595, 0.5343271256807102, 0.5249643991287053, 0.5130648517134513, 0.7580131602578075, 0.7348274620741431, 0.7341801809099809, 0.7074999463902408, 0.9290893112957194, 0.896130362600229, 0.8896360907643062, 0.8667732512123232, 1.0501229826066591, 1.0276121076179003, 1.01304723882372, 0.9917393510831477, 1.161342307568102, 1.1291245609681106, 1.1079392656735292, 1.0938109858524918, 1.2035226235600145, 1.226263478849651, 1.1724755435916903, 1.1553301515952579, 1.2932755854998876, 1.227360989677261, 1.2282904680523223, 1.203004103561663, 1.301298744133899, 1.2818290480161694, 1.2464280682225441, 1.251387892724203 ], "peak_layer": 29, "peak_normalized": 1.301298744133899, "population_cv": 0.24714813814810152, "rest_mean": 8.048455465878187e-05, "spike_contrast": 1.2591821218500583, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010134471231140196, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 4.574834747472778e-05, 4.474666638998315e-05, 4.396259464556351e-05, 4.296607949072495e-05, 6.347901944536716e-05, 6.153735739644617e-05, 6.14831515122205e-05, 5.924884317209944e-05, 7.780561281833798e-05, 7.504550012527034e-05, 7.450164412148297e-05, 7.258701953105628e-05, 8.794145105639473e-05, 8.60563013702631e-05, 8.483658166369423e-05, 8.305217488668859e-05, 9.725539712235332e-05, 9.455735562369227e-05, 9.278321522288024e-05, 9.160005720332265e-05, 0.00010078774357680231, 0.00010269215272273868, 9.818773833103478e-05, 9.675191540736705e-05, 0.00010830401151906699, 0.00010278406261932105, 0.00010286190081387758, 0.00010074432066176087, 0.00010897590254899114, 0.00010734543320722878, 0.00010438081517349929, 0.00010479617048986256 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010133173600479495, "first_to_last_ratio": 0.7712367006907566, "imbalance_abs_log_ratio": 0.25975994774816064, "last_quartile_mean": 0.00013138863323547412, "mean": 0.00011757463107642252, "normalized": [ 0.9240254436843522, 0.8516610763218788, 0.7400083948384197, 0.7305203183107621, 0.9945100925181989, 0.9162256174012862, 0.9029060711458456, 0.8349457218511338, 1.057384774375838, 0.9561549232760007, 0.9255087968738563, 0.9289705112696707, 1.0651337322266232, 1.006401362110571, 0.966928758219108, 0.9306903222204219, 1.1004660046561014, 1.0434523934672617, 1.009934985409439, 0.9657192405683421, 1.061624429236619, 1.0862247001295862, 1.0383026763618062, 1.022368851612807, 1.1429833503196458, 1.0926891991234107, 1.0888718388877934, 1.0734199646869733, 1.177032649101865, 1.140664442905499, 1.1108444025161495, 1.1134249543727335 ], "peak_layer": 29, "peak_normalized": 1.177032649101865, "population_cv": 0.10936728326378568, "rest_mean": 0.00011604396611262389, "spike_contrast": 1.0844184846181808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012584022188093512, "top_five_layers": [ 29, 25, 30, 32, 31 ], "top_five_spike_overlap": 1 }, "values": [ 0.00010864195064641535, 0.00010013373685069382, 8.700621401658282e-05, 8.58906569192186e-05, 0.00011692915722960606, 0.00010772488894872367, 0.0001061588482116349, 9.816843521548435e-05, 0.0001243216247530654, 0.00011241956235608086, 0.00010881635535042733, 0.00010922336514340714, 0.00012523270561359823, 0.00011832726886495948, 0.00011368629202479497, 0.00010942557128146291, 0.0001293868845095858, 0.00012268353020772338, 0.00011874273332068697, 0.00011354408343322575, 0.00012482010060921311, 0.00012771246838383377, 0.0001220780541189015, 0.00012020464055240154, 0.00013438584574032575, 0.0001284725294681266, 0.00012802370474673808, 0.0001262069563381374, 0.00013838917948305607, 0.00013411320105660707, 0.00013060712080914527, 0.00013091052824165672 ] } } } } } }, "step": 0 }, { "modes": { "learned": { "bits_per_byte": 5.18286091123436, "forward": { "activation_sha256": { "attention_branch_output": "5f9ceb4daf4a04de3e80e18a04e2c787acd0577d8e42d246715f712ea2c503dc", "mlp_branch_output": "78bdb1dc451fca8b12adc84e1182b5c9328709671c2cc7151ff16c36d6a3b757", "post_attention_state": "ae4066fc37f2b8857b68206f2c50d90301cff538a7afaed7e0b4959067354e04", "post_mlp_state": "0a445d8d97762f2c9a1505daa03132d58d2d7cce48cebe69f58dcee8b68ddc3c", "pre_attention_input": "bbef604712053bff665de420d5353433a19666047adbe681202c1aa4c9b40d0a", "pre_mlp_input": "4e4a33d69f29ecf2b2276b1faef8a4abb641becd86149078c17228416d2e6c90" }, "logits_sha256": "2d69f7bc604d2338c84b2c31752b4e64390f685f11c1122bb80ad52194829404", "loss_nats": 3.5924854278564453, "mixer_summary_sha256": "57b7f21bb725bbab9d3242fe4186ef8ea689dbc30e549430632de5bdaef4d2ca" }, "loss_nats": 3.5924854278564453, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6920576691627502, "group": 1, "latest_source_mass": 0.47704219818115234, "layer": 1, "max_source_mass": 0.5229578018188477, "mixer_index": 1, "normalized_entropy": 0.9984281673102747, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5229578018188477, "median": 0.5233346819877625, "p05": 0.5158256888389587, "p95": 0.5291492938995361 }, { "label": "current_group_partial", "mean": 0.47704219818115234, "median": 0.47666531801223755, "p05": 0.4708506762981415, "p95": 0.48417431116104126 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6922386288642883, "group": 1, "latest_source_mass": 0.479327529668808, "layer": 2, "max_source_mass": 0.5206724405288696, "mixer_index": 2, "normalized_entropy": 0.9986892369742845, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5206724405288696, "median": 0.5204353928565979, "p05": 0.5125095844268799, "p95": 0.529628336429596 }, { "label": "current_group_partial", "mean": 0.479327529668808, "median": 0.4795646071434021, "p05": 0.47037166357040405, "p95": 0.48749038577079773 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6930604577064514, "group": 1, "latest_source_mass": 0.4976575970649719, "layer": 2, "max_source_mass": 0.5023424625396729, "mixer_index": 3, "normalized_entropy": 0.9998748853693327, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5023424625396729, "median": 0.5015497207641602, "p05": 0.4933284819126129, "p95": 0.5133330225944519 }, { "label": "current_group_partial", "mean": 0.4976575970649719, "median": 0.49845027923583984, "p05": 0.4866669774055481, "p95": 0.5066715478897095 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6924410462379456, "group": 1, "latest_source_mass": 0.48233455419540405, "layer": 3, "max_source_mass": 0.517665445804596, "mixer_index": 4, "normalized_entropy": 0.9989812635154495, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.517665445804596, "median": 0.5173795223236084, "p05": 0.5080559253692627, "p95": 0.528046727180481 }, { "label": "current_group_partial", "mean": 0.48233455419540405, "median": 0.4826204478740692, "p05": 0.4719533324241638, "p95": 0.4919440448284149 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6929901838302612, "group": 1, "latest_source_mass": 0.4986313283443451, "layer": 3, "max_source_mass": 0.5013687014579773, "mixer_index": 5, "normalized_entropy": 0.999773501596649, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5013687014579773, "median": 0.5013427734375, "p05": 0.4864077866077423, "p95": 0.5143497586250305 }, { "label": "current_group_partial", "mean": 0.4986313283443451, "median": 0.4986572265625, "p05": 0.4856501817703247, "p95": 0.5135922431945801 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6929076313972473, "group": 1, "latest_source_mass": 0.49019312858581543, "layer": 4, "max_source_mass": 0.5098068714141846, "mixer_index": 6, "normalized_entropy": 0.9996544036109265, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5098068714141846, "median": 0.5095012187957764, "p05": 0.5024833679199219, "p95": 0.5181499719619751 }, { "label": "current_group_partial", "mean": 0.49019312858581543, "median": 0.49049875140190125, "p05": 0.4818500280380249, "p95": 0.4975166320800781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6927430629730225, "group": 1, "latest_source_mass": 0.4872150421142578, "layer": 4, "max_source_mass": 0.5127849578857422, "mixer_index": 7, "normalized_entropy": 0.9994169815614105, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5127849578857422, "median": 0.5123876333236694, "p05": 0.5027084350585938, "p95": 0.5231767296791077 }, { "label": "current_group_partial", "mean": 0.4872150421142578, "median": 0.48761239647865295, "p05": 0.47682327032089233, "p95": 0.49729159474372864 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6928876638412476, "group": 2, "latest_source_mass": 0.48993486166000366, "layer": 5, "max_source_mass": 0.5100650787353516, "mixer_index": 8, "normalized_entropy": 0.999625596516907, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5100650787353516, "median": 0.5104688405990601, "p05": 0.5006189346313477, "p95": 0.5182909369468689 }, { "label": "completed_group_1", "mean": 0.48993486166000366, "median": 0.48953112959861755, "p05": 0.4817090630531311, "p95": 0.49938106536865234 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.098104476928711, "group": 2, "latest_source_mass": 0.34344637393951416, "layer": 5, "max_source_mass": 0.34344637393951416, "mixer_index": 9, "normalized_entropy": 0.9995377698350576, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.33572152256965637, "median": 0.33574378490448, "p05": 0.3252582252025604, "p95": 0.3447045683860779 }, { "label": "completed_group_1", "mean": 0.32083210349082947, "median": 0.3210340738296509, "p05": 0.3132280111312866, "p95": 0.3294433057308197 }, { "label": "current_group_partial", "mean": 0.34344637393951416, "median": 0.3437310457229614, "p05": 0.335769921541214, "p95": 0.3489701747894287 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098517656326294, "group": 2, "latest_source_mass": 0.33111652731895447, "layer": 6, "max_source_mass": 0.3379058539867401, "mixer_index": 10, "normalized_entropy": 0.9999138619303717, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3379058539867401, "median": 0.33803731203079224, "p05": 0.3307536244392395, "p95": 0.34437015652656555 }, { "label": "completed_group_1", "mean": 0.3309776186943054, "median": 0.33117008209228516, "p05": 0.3264409899711609, "p95": 0.3352589011192322 }, { "label": "current_group_partial", "mean": 0.33111652731895447, "median": 0.3308069109916687, "p05": 0.32785022258758545, "p95": 0.33529508113861084 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0984238386154175, "group": 2, "latest_source_mass": 0.3280537724494934, "layer": 6, "max_source_mass": 0.34072375297546387, "mixer_index": 11, "normalized_entropy": 0.9998284653697795, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34072375297546387, "median": 0.3406665623188019, "p05": 0.33292704820632935, "p95": 0.3486446142196655 }, { "label": "completed_group_1", "mean": 0.3312225341796875, "median": 0.3312021493911743, "p05": 0.32623448967933655, "p95": 0.33618849515914917 }, { "label": "current_group_partial", "mean": 0.3280537724494934, "median": 0.32822102308273315, "p05": 0.323047935962677, "p95": 0.3323698341846466 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0985376834869385, "group": 2, "latest_source_mass": 0.3316706418991089, "layer": 7, "max_source_mass": 0.3372243046760559, "mixer_index": 12, "normalized_entropy": 0.9999320914375883, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3372243046760559, "median": 0.3373582065105438, "p05": 0.33067408204078674, "p95": 0.3436351418495178 }, { "label": "completed_group_1", "mean": 0.33110499382019043, "median": 0.331229567527771, "p05": 0.32648319005966187, "p95": 0.3351142108440399 }, { "label": "current_group_partial", "mean": 0.3316706418991089, "median": 0.3314104676246643, "p05": 0.32845842838287354, "p95": 0.33538299798965454 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0983879566192627, "group": 2, "latest_source_mass": 0.3259334862232208, "layer": 7, "max_source_mass": 0.34103792905807495, "mixer_index": 13, "normalized_entropy": 0.9997958041693498, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.34103792905807495, "median": 0.34118473529815674, "p05": 0.33348870277404785, "p95": 0.3480936288833618 }, { "label": "completed_group_1", "mean": 0.33302855491638184, "median": 0.3329259753227234, "p05": 0.32893261313438416, "p95": 0.33756527304649353 }, { "label": "current_group_partial", "mean": 0.3259334862232208, "median": 0.32614004611968994, "p05": 0.32082775235176086, "p95": 0.33071696758270264 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0984759330749512, "group": 2, "latest_source_mass": 0.3290713131427765, "layer": 8, "max_source_mass": 0.3378061056137085, "mixer_index": 14, "normalized_entropy": 0.9998758837903371, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3378061056137085, "median": 0.3386826515197754, "p05": 0.32766789197921753, "p95": 0.34578830003738403 }, { "label": "completed_group_1", "mean": 0.3331226110458374, "median": 0.3330760598182678, "p05": 0.32885679602622986, "p95": 0.33746954798698425 }, { "label": "current_group_partial", "mean": 0.3290713131427765, "median": 0.3284408450126648, "p05": 0.3237078785896301, "p95": 0.33546704053878784 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0982115268707275, "group": 2, "latest_source_mass": 0.32475554943084717, "layer": 8, "max_source_mass": 0.3427826762199402, "mixer_index": 15, "normalized_entropy": 0.9996352108914892, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3427826762199402, "median": 0.34300696849823, "p05": 0.33116698265075684, "p95": 0.3554348945617676 }, { "label": "completed_group_1", "mean": 0.33246177434921265, "median": 0.3327745795249939, "p05": 0.3261578679084778, "p95": 0.33856457471847534 }, { "label": "current_group_partial", "mean": 0.32475554943084717, "median": 0.32512229681015015, "p05": 0.3160964250564575, "p95": 0.33136001229286194 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098510980606079, "group": 3, "latest_source_mass": 0.32809075713157654, "layer": 9, "max_source_mass": 0.3366020917892456, "mixer_index": 16, "normalized_entropy": 0.9999077854279661, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3366020917892456, "median": 0.33599138259887695, "p05": 0.33130791783332825, "p95": 0.343323677778244 }, { "label": "completed_group_1", "mean": 0.33530712127685547, "median": 0.335271418094635, "p05": 0.33171460032463074, "p95": 0.33948856592178345 }, { "label": "completed_group_2", "mean": 0.32809075713157654, "median": 0.328441321849823, "p05": 0.3236806392669678, "p95": 0.3313118815422058 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3860652446746826, "group": 3, "latest_source_mass": 0.24453917145729065, "layer": 9, "max_source_mass": 0.25563085079193115, "mixer_index": 17, "normalized_entropy": 0.9998347274203562, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25563085079193115, "median": 0.2557356357574463, "p05": 0.24995456635951996, "p95": 0.2610935866832733 }, { "label": "completed_group_1", "mean": 0.25276094675064087, "median": 0.2525555491447449, "p05": 0.24839484691619873, "p95": 0.25784528255462646 }, { "label": "completed_group_2", "mean": 0.24706900119781494, "median": 0.24680209159851074, "p05": 0.24331113696098328, "p95": 0.2516176700592041 }, { "label": "current_group_partial", "mean": 0.24453917145729065, "median": 0.24438345432281494, "p05": 0.24019667506217957, "p95": 0.24867366254329681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3861669301986694, "group": 3, "latest_source_mass": 0.24664369225502014, "layer": 10, "max_source_mass": 0.25406360626220703, "mixer_index": 18, "normalized_entropy": 0.9999080780209492, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25406360626220703, "median": 0.25364670157432556, "p05": 0.24951469898223877, "p95": 0.25949251651763916 }, { "label": "completed_group_1", "mean": 0.25270599126815796, "median": 0.2528834342956543, "p05": 0.24992677569389343, "p95": 0.25502878427505493 }, { "label": "completed_group_2", "mean": 0.24658671021461487, "median": 0.2470710575580597, "p05": 0.2430468648672104, "p95": 0.24908794462680817 }, { "label": "current_group_partial", "mean": 0.24664369225502014, "median": 0.2466917634010315, "p05": 0.24438677728176117, "p95": 0.24867767095565796 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.386000394821167, "group": 3, "latest_source_mass": 0.24240648746490479, "layer": 10, "max_source_mass": 0.2551618814468384, "mixer_index": 19, "normalized_entropy": 0.9997879481393215, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2551618814468384, "median": 0.2564138174057007, "p05": 0.24672001600265503, "p95": 0.2620469629764557 }, { "label": "completed_group_1", "mean": 0.2541123628616333, "median": 0.25428205728530884, "p05": 0.25075286626815796, "p95": 0.2573728561401367 }, { "label": "completed_group_2", "mean": 0.2483193278312683, "median": 0.24777266383171082, "p05": 0.24365577101707458, "p95": 0.25304728746414185 }, { "label": "current_group_partial", "mean": 0.24240648746490479, "median": 0.24237973988056183, "p05": 0.2394917905330658, "p95": 0.2452111691236496 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.386232852935791, "group": 3, "latest_source_mass": 0.24722051620483398, "layer": 11, "max_source_mass": 0.25246676802635193, "mixer_index": 20, "normalized_entropy": 0.9999556312239127, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25183945894241333, "median": 0.2516742944717407, "p05": 0.2478763908147812, "p95": 0.2557043135166168 }, { "label": "completed_group_1", "mean": 0.25246676802635193, "median": 0.2523234784603119, "p05": 0.25062504410743713, "p95": 0.254791259765625 }, { "label": "completed_group_2", "mean": 0.24847325682640076, "median": 0.24844470620155334, "p05": 0.24627941846847534, "p95": 0.250301718711853 }, { "label": "current_group_partial", "mean": 0.24722051620483398, "median": 0.24719130992889404, "p05": 0.2453235685825348, "p95": 0.24936428666114807 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.385960340499878, "group": 3, "latest_source_mass": 0.24082177877426147, "layer": 11, "max_source_mass": 0.25490617752075195, "mixer_index": 21, "normalized_entropy": 0.9997590550539766, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25454509258270264, "median": 0.2548168897628784, "p05": 0.24703091382980347, "p95": 0.2618085741996765 }, { "label": "completed_group_1", "mean": 0.25490617752075195, "median": 0.2547938823699951, "p05": 0.2522759437561035, "p95": 0.2576809227466583 }, { "label": "completed_group_2", "mean": 0.24972698092460632, "median": 0.2496279776096344, "p05": 0.24634553492069244, "p95": 0.25290369987487793 }, { "label": "current_group_partial", "mean": 0.24082177877426147, "median": 0.24057629704475403, "p05": 0.23598811030387878, "p95": 0.245779350399971 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.386134147644043, "group": 3, "latest_source_mass": 0.24431857466697693, "layer": 12, "max_source_mass": 0.25437670946121216, "mixer_index": 22, "normalized_entropy": 0.9998844304064556, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25200381875038147, "median": 0.25220245122909546, "p05": 0.2462744265794754, "p95": 0.25811436772346497 }, { "label": "completed_group_1", "mean": 0.25437670946121216, "median": 0.2543410062789917, "p05": 0.25215157866477966, "p95": 0.25676217675209045 }, { "label": "completed_group_2", "mean": 0.24930089712142944, "median": 0.24924488365650177, "p05": 0.24625414609909058, "p95": 0.2522015869617462 }, { "label": "current_group_partial", "mean": 0.24431857466697693, "median": 0.24446552991867065, "p05": 0.24055814743041992, "p95": 0.24748677015304565 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3859115839004517, "group": 3, "latest_source_mass": 0.2395908236503601, "layer": 12, "max_source_mass": 0.2563633620738983, "mixer_index": 23, "normalized_entropy": 0.9997238846018751, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2541099786758423, "median": 0.25436341762542725, "p05": 0.2479730099439621, "p95": 0.2596602141857147 }, { "label": "completed_group_1", "mean": 0.2563633620738983, "median": 0.25615423917770386, "p05": 0.2534457743167877, "p95": 0.26022869348526 }, { "label": "completed_group_2", "mean": 0.2499358206987381, "median": 0.24995511770248413, "p05": 0.247016042470932, "p95": 0.2527387738227844 }, { "label": "current_group_partial", "mean": 0.2395908236503601, "median": 0.24023187160491943, "p05": 0.23504012823104858, "p95": 0.24278664588928223 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3861732482910156, "group": 4, "latest_source_mass": 0.24519044160842896, "layer": 13, "max_source_mass": 0.25510814785957336, "mixer_index": 24, "normalized_entropy": 0.999912635561197, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2492598295211792, "median": 0.24941539764404297, "p05": 0.24585920572280884, "p95": 0.2526070773601532 }, { "label": "completed_group_1", "mean": 0.25510814785957336, "median": 0.25510847568511963, "p05": 0.253096342086792, "p95": 0.2572459578514099 }, { "label": "completed_group_2", "mean": 0.2504415512084961, "median": 0.2503618597984314, "p05": 0.24890245497226715, "p95": 0.2520940899848938 }, { "label": "completed_group_3", "mean": 0.24519044160842896, "median": 0.24486954510211945, "p05": 0.2423042356967926, "p95": 0.24860209226608276 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6091704368591309, "group": 4, "latest_source_mass": 0.19571462273597717, "layer": 13, "max_source_mass": 0.20561829209327698, "mixer_index": 25, "normalized_entropy": 0.9998338080811301, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20294126868247986, "median": 0.20319771766662598, "p05": 0.19784998893737793, "p95": 0.20785459876060486 }, { "label": "completed_group_1", "mean": 0.20561829209327698, "median": 0.2055630385875702, "p05": 0.203423410654068, "p95": 0.20820140838623047 }, { "label": "completed_group_2", "mean": 0.20076946914196014, "median": 0.20075580477714539, "p05": 0.19847290217876434, "p95": 0.2027682065963745 }, { "label": "completed_group_3", "mean": 0.19495636224746704, "median": 0.19467401504516602, "p05": 0.19074591994285583, "p95": 0.19898995757102966 }, { "label": "current_group_partial", "mean": 0.19571462273597717, "median": 0.19596168398857117, "p05": 0.19247503578662872, "p95": 0.19751161336898804 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093817949295044, "group": 4, "latest_source_mass": 0.2000274658203125, "layer": 14, "max_source_mass": 0.20290037989616394, "mixer_index": 26, "normalized_entropy": 0.9999651322339542, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1999536156654358, "median": 0.1995687186717987, "p05": 0.19656820595264435, "p95": 0.20360995829105377 }, { "label": "completed_group_1", "mean": 0.20290037989616394, "median": 0.20293280482292175, "p05": 0.20150557160377502, "p95": 0.204205721616745 }, { "label": "completed_group_2", "mean": 0.19905918836593628, "median": 0.19910168647766113, "p05": 0.19694079458713531, "p95": 0.2005346566438675 }, { "label": "completed_group_3", "mean": 0.1980593204498291, "median": 0.1983218491077423, "p05": 0.19644758105278015, "p95": 0.19932645559310913 }, { "label": "current_group_partial", "mean": 0.2000274658203125, "median": 0.19992563128471375, "p05": 0.19864776730537415, "p95": 0.20164194703102112 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6089024543762207, "group": 4, "latest_source_mass": 0.1908060908317566, "layer": 14, "max_source_mass": 0.20689502358436584, "mixer_index": 27, "normalized_entropy": 0.999667301202648, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20434537529945374, "median": 0.2042810022830963, "p05": 0.19885064661502838, "p95": 0.21045605838298798 }, { "label": "completed_group_1", "mean": 0.20689502358436584, "median": 0.20666636526584625, "p05": 0.20447728037834167, "p95": 0.21027149260044098 }, { "label": "completed_group_2", "mean": 0.20309436321258545, "median": 0.203084334731102, "p05": 0.20090451836585999, "p95": 0.20521050691604614 }, { "label": "completed_group_3", "mean": 0.19485914707183838, "median": 0.19490095973014832, "p05": 0.19095906615257263, "p95": 0.19829493761062622 }, { "label": "current_group_partial", "mean": 0.1908060908317566, "median": 0.19071418046951294, "p05": 0.18832355737686157, "p95": 0.1931709200143814 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.609398365020752, "group": 4, "latest_source_mass": 0.19768188893795013, "layer": 15, "max_source_mass": 0.2025333046913147, "mixer_index": 28, "normalized_entropy": 0.9999754278105152, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20015040040016174, "median": 0.20033058524131775, "p05": 0.19777344167232513, "p95": 0.20235712826251984 }, { "label": "completed_group_1", "mean": 0.2025333046913147, "median": 0.2024737000465393, "p05": 0.20170874893665314, "p95": 0.20358291268348694 }, { "label": "completed_group_2", "mean": 0.2003234475851059, "median": 0.20028962194919586, "p05": 0.1992899477481842, "p95": 0.20118050277233124 }, { "label": "completed_group_3", "mean": 0.19931095838546753, "median": 0.1992638111114502, "p05": 0.19840562343597412, "p95": 0.2002609521150589 }, { "label": "current_group_partial", "mean": 0.19768188893795013, "median": 0.19755831360816956, "p05": 0.1966717392206192, "p95": 0.19881953299045563 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6085304021835327, "group": 4, "latest_source_mass": 0.18813428282737732, "layer": 15, "max_source_mass": 0.21047377586364746, "mixer_index": 29, "normalized_entropy": 0.9994361321778514, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20286625623703003, "median": 0.20224162936210632, "p05": 0.19363349676132202, "p95": 0.21225287020206451 }, { "label": "completed_group_1", "mean": 0.21047377586364746, "median": 0.2103899121284485, "p05": 0.20706231892108917, "p95": 0.2142515927553177 }, { "label": "completed_group_2", "mean": 0.20388861000537872, "median": 0.2042773962020874, "p05": 0.2002551406621933, "p95": 0.20698434114456177 }, { "label": "completed_group_3", "mean": 0.19463706016540527, "median": 0.194687619805336, "p05": 0.1895509660243988, "p95": 0.19856791198253632 }, { "label": "current_group_partial", "mean": 0.18813428282737732, "median": 0.18814176321029663, "p05": 0.18432703614234924, "p95": 0.19110313057899475 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.60928213596344, "group": 4, "latest_source_mass": 0.19474157691001892, "layer": 16, "max_source_mass": 0.20464840531349182, "mixer_index": 30, "normalized_entropy": 0.9999032106367963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20026026666164398, "median": 0.20021198689937592, "p05": 0.19762656092643738, "p95": 0.20328035950660706 }, { "label": "completed_group_1", "mean": 0.20464840531349182, "median": 0.20486173033714294, "p05": 0.20261231064796448, "p95": 0.20639576017856598 }, { "label": "completed_group_2", "mean": 0.20178081095218658, "median": 0.20188090205192566, "p05": 0.2006548047065735, "p95": 0.2025863528251648 }, { "label": "completed_group_3", "mean": 0.1985689103603363, "median": 0.19860434532165527, "p05": 0.19688552618026733, "p95": 0.20015479624271393 }, { "label": "current_group_partial", "mean": 0.19474157691001892, "median": 0.19454869627952576, "p05": 0.19281607866287231, "p95": 0.1971154808998108 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6085617542266846, "group": 4, "latest_source_mass": 0.18782781064510345, "layer": 16, "max_source_mass": 0.2110326886177063, "mixer_index": 31, "normalized_entropy": 0.9994556122975314, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20295968651771545, "median": 0.20242132246494293, "p05": 0.1979435831308365, "p95": 0.209271639585495 }, { "label": "completed_group_1", "mean": 0.2110326886177063, "median": 0.21132414042949677, "p05": 0.20678338408470154, "p95": 0.2153906226158142 }, { "label": "completed_group_2", "mean": 0.20332126319408417, "median": 0.20331284403800964, "p05": 0.2010943740606308, "p95": 0.2055126279592514 }, { "label": "completed_group_3", "mean": 0.194858580827713, "median": 0.19520977139472961, "p05": 0.190799281001091, "p95": 0.1979817897081375 }, { "label": "current_group_partial", "mean": 0.18782781064510345, "median": 0.18755686283111572, "p05": 0.18426461517810822, "p95": 0.19186276197433472 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093013286590576, "group": 5, "latest_source_mass": 0.19532112777233124, "layer": 17, "max_source_mass": 0.20404377579689026, "mixer_index": 32, "normalized_entropy": 0.999915135729072, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20157882571220398, "median": 0.20241472125053406, "p05": 0.196755051612854, "p95": 0.20509284734725952 }, { "label": "completed_group_1", "mean": 0.20404377579689026, "median": 0.20405352115631104, "p05": 0.2024468630552292, "p95": 0.20564454793930054 }, { "label": "completed_group_2", "mean": 0.20028471946716309, "median": 0.20018786191940308, "p05": 0.19867847859859467, "p95": 0.20205119252204895 }, { "label": "completed_group_3", "mean": 0.19877153635025024, "median": 0.19872286915779114, "p05": 0.19718100130558014, "p95": 0.20058035850524902 }, { "label": "completed_group_4", "mean": 0.19532112777233124, "median": 0.1952555775642395, "p05": 0.19347216188907623, "p95": 0.19731438159942627 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.79081130027771, "group": 5, "latest_source_mass": 0.16283263266086578, "layer": 17, "max_source_mass": 0.17698298394680023, "mixer_index": 33, "normalized_entropy": 0.9994708168330465, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1695082038640976, "median": 0.17041431367397308, "p05": 0.1590532660484314, "p95": 0.17703557014465332 }, { "label": "completed_group_1", "mean": 0.17698298394680023, "median": 0.17722181975841522, "p05": 0.17372196912765503, "p95": 0.1802559494972229 }, { "label": "completed_group_2", "mean": 0.17063109576702118, "median": 0.17032179236412048, "p05": 0.1679578572511673, "p95": 0.173906147480011 }, { "label": "completed_group_3", "mean": 0.16265574097633362, "median": 0.162393718957901, "p05": 0.1576516330242157, "p95": 0.16904693841934204 }, { "label": "completed_group_4", "mean": 0.15738935768604279, "median": 0.15682470798492432, "p05": 0.15282297134399414, "p95": 0.16380727291107178 }, { "label": "current_group_partial", "mean": 0.16283263266086578, "median": 0.16261553764343262, "p05": 0.1596994698047638, "p95": 0.16662785410881042 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917133569717407, "group": 5, "latest_source_mass": 0.16537019610404968, "layer": 18, "max_source_mass": 0.1689736545085907, "mixer_index": 34, "normalized_entropy": 0.9999742642597368, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16738522052764893, "median": 0.1673279106616974, "p05": 0.16558583080768585, "p95": 0.1691284030675888 }, { "label": "completed_group_1", "mean": 0.1689736545085907, "median": 0.1689835786819458, "p05": 0.1681547313928604, "p95": 0.1699315309524536 }, { "label": "completed_group_2", "mean": 0.16775795817375183, "median": 0.167745441198349, "p05": 0.16722241044044495, "p95": 0.16836081445217133 }, { "label": "completed_group_3", "mean": 0.16576024889945984, "median": 0.16582953929901123, "p05": 0.1648658663034439, "p95": 0.16642220318317413 }, { "label": "completed_group_4", "mean": 0.16475272178649902, "median": 0.16466641426086426, "p05": 0.16412994265556335, "p95": 0.16559217870235443 }, { "label": "current_group_partial", "mean": 0.16537019610404968, "median": 0.16532129049301147, "p05": 0.16465029120445251, "p95": 0.16635482013225555 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7902145385742188, "group": 5, "latest_source_mass": 0.15478186309337616, "layer": 18, "max_source_mass": 0.18038460612297058, "mixer_index": 35, "normalized_entropy": 0.9991377577848093, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.17057639360427856, "median": 0.17081734538078308, "p05": 0.16390566527843475, "p95": 0.17745791375637054 }, { "label": "completed_group_1", "mean": 0.18038460612297058, "median": 0.18108245730400085, "p05": 0.17430555820465088, "p95": 0.18539397418498993 }, { "label": "completed_group_2", "mean": 0.17244961857795715, "median": 0.17250855267047882, "p05": 0.17035990953445435, "p95": 0.17447513341903687 }, { "label": "completed_group_3", "mean": 0.164292573928833, "median": 0.16445159912109375, "p05": 0.16052639484405518, "p95": 0.167647585272789 }, { "label": "completed_group_4", "mean": 0.15751495957374573, "median": 0.1570037603378296, "p05": 0.1535545289516449, "p95": 0.16239827871322632 }, { "label": "current_group_partial", "mean": 0.15478186309337616, "median": 0.1544024646282196, "p05": 0.15216419100761414, "p95": 0.15823917090892792 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917354106903076, "group": 5, "latest_source_mass": 0.165629044175148, "layer": 19, "max_source_mass": 0.16850554943084717, "mixer_index": 36, "normalized_entropy": 0.9999865726744239, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.16660895943641663, "median": 0.1668481081724167, "p05": 0.1639198660850525, "p95": 0.16857405006885529 }, { "label": "completed_group_1", "mean": 0.16850554943084717, "median": 0.16847547888755798, "p05": 0.16779136657714844, "p95": 0.16932576894760132 }, { "label": "completed_group_2", "mean": 0.1666441708803177, "median": 0.1666129231452942, "p05": 0.1657959520816803, "p95": 0.16746148467063904 }, { "label": "completed_group_3", "mean": 0.16635674238204956, "median": 0.1662396490573883, "p05": 0.16566412150859833, "p95": 0.16728293895721436 }, { "label": "completed_group_4", "mean": 0.16625551879405975, "median": 0.16612455248832703, "p05": 0.1655116081237793, "p95": 0.16721341013908386 }, { "label": "current_group_partial", "mean": 0.165629044175148, "median": 0.16562071442604065, "p05": 0.16496792435646057, "p95": 0.16613325476646423 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7898752689361572, "group": 5, "latest_source_mass": 0.15259043872356415, "layer": 19, "max_source_mass": 0.1802286058664322, "mixer_index": 37, "normalized_entropy": 0.9989484077945409, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.17184747755527496, "median": 0.171621635556221, "p05": 0.16667789220809937, "p95": 0.17803548276424408 }, { "label": "completed_group_1", "mean": 0.1802286058664322, "median": 0.1802484691143036, "p05": 0.17482617497444153, "p95": 0.18618299067020416 }, { "label": "completed_group_2", "mean": 0.17470473051071167, "median": 0.17439083755016327, "p05": 0.17253704369068146, "p95": 0.17793725430965424 }, { "label": "completed_group_3", "mean": 0.16429948806762695, "median": 0.16437724232673645, "p05": 0.16081778705120087, "p95": 0.1669623851776123 }, { "label": "completed_group_4", "mean": 0.15632927417755127, "median": 0.15611344575881958, "p05": 0.15268553793430328, "p95": 0.16074182093143463 }, { "label": "current_group_partial", "mean": 0.15259043872356415, "median": 0.15249013900756836, "p05": 0.14960677921772003, "p95": 0.1559593677520752 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917178869247437, "group": 5, "latest_source_mass": 0.16608723998069763, "layer": 20, "max_source_mass": 0.16851314902305603, "mixer_index": 38, "normalized_entropy": 0.9999767924746454, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1659121811389923, "median": 0.1657598316669464, "p05": 0.16222882270812988, "p95": 0.16958965361118317 }, { "label": "completed_group_1", "mean": 0.16851314902305603, "median": 0.16853493452072144, "p05": 0.1675589680671692, "p95": 0.1694892942905426 }, { "label": "completed_group_2", "mean": 0.16693057119846344, "median": 0.16706672310829163, "p05": 0.1656341552734375, "p95": 0.16794949769973755 }, { "label": "completed_group_3", "mean": 0.16679996252059937, "median": 0.16667509078979492, "p05": 0.16564559936523438, "p95": 0.16818198561668396 }, { "label": "completed_group_4", "mean": 0.16575691103935242, "median": 0.16555160284042358, "p05": 0.16459843516349792, "p95": 0.16740773618221283 }, { "label": "current_group_partial", "mean": 0.16608723998069763, "median": 0.1660577356815338, "p05": 0.16539554297924042, "p95": 0.1667378544807434 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7883799076080322, "group": 5, "latest_source_mass": 0.1484336256980896, "layer": 20, "max_source_mass": 0.18512636423110962, "mixer_index": 39, "normalized_entropy": 0.9981138307467806, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1728566437959671, "median": 0.17455406486988068, "p05": 0.1564135104417801, "p95": 0.1859586089849472 }, { "label": "completed_group_1", "mean": 0.18512636423110962, "median": 0.18562474846839905, "p05": 0.1771477460861206, "p95": 0.1933506578207016 }, { "label": "completed_group_2", "mean": 0.17585304379463196, "median": 0.17591087520122528, "p05": 0.1723366677761078, "p95": 0.17899538576602936 }, { "label": "completed_group_3", "mean": 0.16398006677627563, "median": 0.16363000869750977, "p05": 0.15788918733596802, "p95": 0.17107567191123962 }, { "label": "completed_group_4", "mean": 0.1537502557039261, "median": 0.1523861587047577, "p05": 0.1478816270828247, "p95": 0.16301551461219788 }, { "label": "current_group_partial", "mean": 0.1484336256980896, "median": 0.14806321263313293, "p05": 0.1446857899427414, "p95": 0.1533387005329132 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917424440383911, "group": 6, "latest_source_mass": 0.16565825045108795, "layer": 21, "max_source_mass": 0.1680906116962433, "mixer_index": 40, "normalized_entropy": 0.9999904980607296, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1667865514755249, "median": 0.16677325963974, "p05": 0.16519494354724884, "p95": 0.1684957593679428 }, { "label": "completed_group_1", "mean": 0.1680906116962433, "median": 0.16817617416381836, "p05": 0.16728028655052185, "p95": 0.16889086365699768 }, { "label": "completed_group_2", "mean": 0.16619735956192017, "median": 0.1661883294582367, "p05": 0.1652708500623703, "p95": 0.1671273112297058 }, { "label": "completed_group_3", "mean": 0.16709285974502563, "median": 0.16709713637828827, "p05": 0.1664952039718628, "p95": 0.16763292253017426 }, { "label": "completed_group_4", "mean": 0.16617438197135925, "median": 0.16625460982322693, "p05": 0.16550303995609283, "p95": 0.1667831838130951 }, { "label": "completed_group_5", "mean": 0.16565825045108795, "median": 0.16577762365341187, "p05": 0.16497491300106049, "p95": 0.16629165410995483 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9426109790802002, "group": 6, "latest_source_mass": 0.12981724739074707, "layer": 21, "max_source_mass": 0.16000477969646454, "mixer_index": 41, "normalized_entropy": 0.9983045620185934, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14892546832561493, "median": 0.14995476603507996, "p05": 0.1375032216310501, "p95": 0.158592090010643 }, { "label": "completed_group_1", "mean": 0.16000477969646454, "median": 0.16008985042572021, "p05": 0.153324156999588, "p95": 0.16691897809505463 }, { "label": "completed_group_2", "mean": 0.15354624390602112, "median": 0.1537785828113556, "p05": 0.14996647834777832, "p95": 0.1563865840435028 }, { "label": "completed_group_3", "mean": 0.14328594505786896, "median": 0.14265790581703186, "p05": 0.13877804577350616, "p95": 0.1482013314962387 }, { "label": "completed_group_4", "mean": 0.13467344641685486, "median": 0.13396033644676208, "p05": 0.13011276721954346, "p95": 0.14165091514587402 }, { "label": "completed_group_5", "mean": 0.12974688410758972, "median": 0.12941759824752808, "p05": 0.1261180341243744, "p95": 0.13459308445453644 }, { "label": "current_group_partial", "mean": 0.12981724739074707, "median": 0.12967640161514282, "p05": 0.1270051896572113, "p95": 0.13326594233512878 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458978176116943, "group": 6, "latest_source_mass": 0.143035888671875, "layer": 22, "max_source_mass": 0.1434222161769867, "mixer_index": 42, "normalized_entropy": 0.9999936628915652, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1419999748468399, "median": 0.1419375240802765, "p05": 0.14045022428035736, "p95": 0.14377202093601227 }, { "label": "completed_group_1", "mean": 0.1434222161769867, "median": 0.14351919293403625, "p05": 0.14237116277217865, "p95": 0.1442951261997223 }, { "label": "completed_group_2", "mean": 0.14279979467391968, "median": 0.1427600383758545, "p05": 0.14219357073307037, "p95": 0.14337173104286194 }, { "label": "completed_group_3", "mean": 0.14312142133712769, "median": 0.14313778281211853, "p05": 0.14236494898796082, "p95": 0.14370372891426086 }, { "label": "completed_group_4", "mean": 0.14279043674468994, "median": 0.14272844791412354, "p05": 0.14216938614845276, "p95": 0.14356710016727448 }, { "label": "completed_group_5", "mean": 0.1428302675485611, "median": 0.14281237125396729, "p05": 0.1421329379081726, "p95": 0.1434640884399414 }, { "label": "current_group_partial", "mean": 0.143035888671875, "median": 0.14297914505004883, "p05": 0.14242705702781677, "p95": 0.14364232122898102 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9415357112884521, "group": 6, "latest_source_mass": 0.1259511113166809, "layer": 22, "max_source_mass": 0.16313126683235168, "mixer_index": 43, "normalized_entropy": 0.9977519836828105, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1496729701757431, "median": 0.15015238523483276, "p05": 0.13863591849803925, "p95": 0.15954872965812683 }, { "label": "completed_group_1", "mean": 0.16313126683235168, "median": 0.163755863904953, "p05": 0.15348099172115326, "p95": 0.17192509770393372 }, { "label": "completed_group_2", "mean": 0.15414540469646454, "median": 0.1541890650987625, "p05": 0.1515030711889267, "p95": 0.15697401762008667 }, { "label": "completed_group_3", "mean": 0.1440294086933136, "median": 0.14383843541145325, "p05": 0.13977289199829102, "p95": 0.14864303171634674 }, { "label": "completed_group_4", "mean": 0.1343868225812912, "median": 0.13351139426231384, "p05": 0.12925155460834503, "p95": 0.1419946253299713 }, { "label": "completed_group_5", "mean": 0.12868301570415497, "median": 0.12839242815971375, "p05": 0.12411807477474213, "p95": 0.13453525304794312 }, { "label": "current_group_partial", "mean": 0.1259511113166809, "median": 0.1258702129125595, "p05": 0.12331026792526245, "p95": 0.12794387340545654 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458801746368408, "group": 6, "latest_source_mass": 0.14443498849868774, "layer": 23, "max_source_mass": 0.14443498849868774, "mixer_index": 44, "normalized_entropy": 0.9999845961960335, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1429852843284607, "median": 0.1430346667766571, "p05": 0.14120227098464966, "p95": 0.1445307582616806 }, { "label": "completed_group_1", "mean": 0.1429750919342041, "median": 0.14302998781204224, "p05": 0.14226141571998596, "p95": 0.14353078603744507 }, { "label": "completed_group_2", "mean": 0.14214137196540833, "median": 0.14214855432510376, "p05": 0.14139141142368317, "p95": 0.14300128817558289 }, { "label": "completed_group_3", "mean": 0.1414642482995987, "median": 0.14139702916145325, "p05": 0.14092163741588593, "p95": 0.14213398098945618 }, { "label": "completed_group_4", "mean": 0.1421438455581665, "median": 0.14223212003707886, "p05": 0.141431987285614, "p95": 0.14265216886997223 }, { "label": "completed_group_5", "mean": 0.14385518431663513, "median": 0.1438562422990799, "p05": 0.14321480691432953, "p95": 0.14455664157867432 }, { "label": "current_group_partial", "mean": 0.14443498849868774, "median": 0.14441588521003723, "p05": 0.14399737119674683, "p95": 0.1448911726474762 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9424511194229126, "group": 6, "latest_source_mass": 0.12740188837051392, "layer": 23, "max_source_mass": 0.16045089066028595, "mixer_index": 45, "normalized_entropy": 0.9982224104057015, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14928652346134186, "median": 0.1492544412612915, "p05": 0.14078925549983978, "p95": 0.15894511342048645 }, { "label": "completed_group_1", "mean": 0.16045089066028595, "median": 0.16077518463134766, "p05": 0.15384821593761444, "p95": 0.16740038990974426 }, { "label": "completed_group_2", "mean": 0.15314346551895142, "median": 0.1533186435699463, "p05": 0.15041443705558777, "p95": 0.15549692511558533 }, { "label": "completed_group_3", "mean": 0.1440504640340805, "median": 0.14415611326694489, "p05": 0.13973577320575714, "p95": 0.1474868506193161 }, { "label": "completed_group_4", "mean": 0.1356543004512787, "median": 0.13497784733772278, "p05": 0.13136392831802368, "p95": 0.14105865359306335 }, { "label": "completed_group_5", "mean": 0.13001246750354767, "median": 0.13024744391441345, "p05": 0.126283660531044, "p95": 0.1334703266620636 }, { "label": "current_group_partial", "mean": 0.12740188837051392, "median": 0.1274060606956482, "p05": 0.12529851496219635, "p95": 0.12901271879673004 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458882808685303, "group": 6, "latest_source_mass": 0.1432376205921173, "layer": 24, "max_source_mass": 0.14391477406024933, "mixer_index": 46, "normalized_entropy": 0.9999887619750616, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14319148659706116, "median": 0.14321468770503998, "p05": 0.14192207157611847, "p95": 0.1444668471813202 }, { "label": "completed_group_1", "mean": 0.14391477406024933, "median": 0.1440032571554184, "p05": 0.14299027621746063, "p95": 0.14467763900756836 }, { "label": "completed_group_2", "mean": 0.14284169673919678, "median": 0.14285600185394287, "p05": 0.14201918244361877, "p95": 0.1436670869588852 }, { "label": "completed_group_3", "mean": 0.1420421302318573, "median": 0.1421079933643341, "p05": 0.1414509415626526, "p95": 0.14248287677764893 }, { "label": "completed_group_4", "mean": 0.14134714007377625, "median": 0.14131714403629303, "p05": 0.14094975590705872, "p95": 0.14189478754997253 }, { "label": "completed_group_5", "mean": 0.14342516660690308, "median": 0.14335885643959045, "p05": 0.14284199476242065, "p95": 0.14415055513381958 }, { "label": "current_group_partial", "mean": 0.1432376205921173, "median": 0.14320111274719238, "p05": 0.14274808764457703, "p95": 0.1438399851322174 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9395697116851807, "group": 6, "latest_source_mass": 0.12089881300926208, "layer": 24, "max_source_mass": 0.16719450056552887, "mixer_index": 47, "normalized_entropy": 0.9967416597455897, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15016257762908936, "median": 0.15012264251708984, "p05": 0.13872724771499634, "p95": 0.1616549789905548 }, { "label": "completed_group_1", "mean": 0.16719450056552887, "median": 0.1678844839334488, "p05": 0.15553203225135803, "p95": 0.1781744658946991 }, { "label": "completed_group_2", "mean": 0.15599417686462402, "median": 0.15601703524589539, "p05": 0.15278863906860352, "p95": 0.15932069718837738 }, { "label": "completed_group_3", "mean": 0.14574658870697021, "median": 0.14581912755966187, "p05": 0.14079399406909943, "p95": 0.15032999217510223 }, { "label": "completed_group_4", "mean": 0.13377559185028076, "median": 0.1329595148563385, "p05": 0.12759888172149658, "p95": 0.14201229810714722 }, { "label": "completed_group_5", "mean": 0.12622776627540588, "median": 0.12609627842903137, "p05": 0.1209360659122467, "p95": 0.13270525634288788 }, { "label": "current_group_partial", "mean": 0.12089881300926208, "median": 0.1206001490354538, "p05": 0.1178487241268158, "p95": 0.12392976135015488 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458978176116943, "group": 7, "latest_source_mass": 0.14348836243152618, "layer": 25, "max_source_mass": 0.14348836243152618, "mixer_index": 48, "normalized_entropy": 0.9999936628915652, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428530067205429, "median": 0.14293229579925537, "p05": 0.14151178300380707, "p95": 0.14395615458488464 }, { "label": "completed_group_1", "mean": 0.14342239499092102, "median": 0.1434272974729538, "p05": 0.14267732203006744, "p95": 0.14410075545310974 }, { "label": "completed_group_2", "mean": 0.14335417747497559, "median": 0.14335879683494568, "p05": 0.1428845077753067, "p95": 0.14383281767368317 }, { "label": "completed_group_3", "mean": 0.14274315536022186, "median": 0.14269700646400452, "p05": 0.1423313021659851, "p95": 0.14328503608703613 }, { "label": "completed_group_4", "mean": 0.1419316530227661, "median": 0.14191937446594238, "p05": 0.14147570729255676, "p95": 0.14250396192073822 }, { "label": "completed_group_5", "mean": 0.14220724999904633, "median": 0.14221754670143127, "p05": 0.14166903495788574, "p95": 0.1429031491279602 }, { "label": "completed_group_6", "mean": 0.14348836243152618, "median": 0.14348852634429932, "p05": 0.14303231239318848, "p95": 0.14401927590370178 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.074862480163574, "group": 7, "latest_source_mass": 0.11496111750602722, "layer": 25, "max_source_mass": 0.14486080408096313, "mixer_index": 49, "normalized_entropy": 0.997797936886188, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13159659504890442, "median": 0.13219013810157776, "p05": 0.12017668038606644, "p95": 0.14146660268306732 }, { "label": "completed_group_1", "mean": 0.14486080408096313, "median": 0.14564016461372375, "p05": 0.13528235256671906, "p95": 0.15322431921958923 }, { "label": "completed_group_2", "mean": 0.13573890924453735, "median": 0.1359761655330658, "p05": 0.13284167647361755, "p95": 0.13837191462516785 }, { "label": "completed_group_3", "mean": 0.12842535972595215, "median": 0.1282735913991928, "p05": 0.12434759736061096, "p95": 0.13281521201133728 }, { "label": "completed_group_4", "mean": 0.12030278891324997, "median": 0.12004269659519196, "p05": 0.11514662206172943, "p95": 0.12723737955093384 }, { "label": "completed_group_5", "mean": 0.11495959758758545, "median": 0.11503666639328003, "p05": 0.11052325367927551, "p95": 0.12032671272754669 }, { "label": "completed_group_6", "mean": 0.1091548278927803, "median": 0.1089196503162384, "p05": 0.10650476068258286, "p95": 0.11220317333936691 }, { "label": "current_group_partial", "mean": 0.11496111750602722, "median": 0.11521677672863007, "p05": 0.11175674200057983, "p95": 0.11837965995073318 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079418659210205, "group": 7, "latest_source_mass": 0.12366288155317307, "layer": 26, "max_source_mass": 0.1261361837387085, "mixer_index": 50, "normalized_entropy": 0.9999889958581801, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12459750473499298, "median": 0.12467166036367416, "p05": 0.12324866652488708, "p95": 0.12578506767749786 }, { "label": "completed_group_1", "mean": 0.12504151463508606, "median": 0.12500853836536407, "p05": 0.12452991306781769, "p95": 0.12561221420764923 }, { "label": "completed_group_2", "mean": 0.12489250302314758, "median": 0.12485124170780182, "p05": 0.12434293329715729, "p95": 0.1255818009376526 }, { "label": "completed_group_3", "mean": 0.1250421404838562, "median": 0.12497402727603912, "p05": 0.12459105253219604, "p95": 0.12559659779071808 }, { "label": "completed_group_4", "mean": 0.12461285293102264, "median": 0.12454816699028015, "p05": 0.12421505898237228, "p95": 0.1250859797000885 }, { "label": "completed_group_5", "mean": 0.12601442635059357, "median": 0.12595048546791077, "p05": 0.1256113201379776, "p95": 0.12648876011371613 }, { "label": "completed_group_6", "mean": 0.1261361837387085, "median": 0.1261894702911377, "p05": 0.1256546974182129, "p95": 0.12649866938591003 }, { "label": "current_group_partial", "mean": 0.12366288155317307, "median": 0.12370622158050537, "p05": 0.12311097979545593, "p95": 0.12407678365707397 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0741164684295654, "group": 7, "latest_source_mass": 0.11192375421524048, "layer": 26, "max_source_mass": 0.14626389741897583, "mixer_index": 51, "normalized_entropy": 0.9974391810764882, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1323997974395752, "median": 0.1329696774482727, "p05": 0.12004873901605606, "p95": 0.14371390640735626 }, { "label": "completed_group_1", "mean": 0.14626389741897583, "median": 0.1470441222190857, "p05": 0.1354091465473175, "p95": 0.1561085730791092 }, { "label": "completed_group_2", "mean": 0.13691633939743042, "median": 0.13697296380996704, "p05": 0.13360059261322021, "p95": 0.14020705223083496 }, { "label": "completed_group_3", "mean": 0.12836162745952606, "median": 0.12827181816101074, "p05": 0.12401524186134338, "p95": 0.13266678154468536 }, { "label": "completed_group_4", "mean": 0.12002508342266083, "median": 0.11955710500478745, "p05": 0.11444184184074402, "p95": 0.12734319269657135 }, { "label": "completed_group_5", "mean": 0.11502262204885483, "median": 0.11501411348581314, "p05": 0.1104419082403183, "p95": 0.12084008753299713 }, { "label": "completed_group_6", "mean": 0.10908685624599457, "median": 0.10856537520885468, "p05": 0.10614658892154694, "p95": 0.11223960667848587 }, { "label": "current_group_partial", "mean": 0.11192375421524048, "median": 0.11169644445180893, "p05": 0.10886920988559723, "p95": 0.11444390565156937 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0793213844299316, "group": 7, "latest_source_mass": 0.12797673046588898, "layer": 27, "max_source_mass": 0.12797673046588898, "mixer_index": 52, "normalized_entropy": 0.9999422165771455, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12429114431142807, "median": 0.12408939003944397, "p05": 0.12283186614513397, "p95": 0.12610697746276855 }, { "label": "completed_group_1", "mean": 0.12243181467056274, "median": 0.12224709987640381, "p05": 0.12128561735153198, "p95": 0.12399756163358688 }, { "label": "completed_group_2", "mean": 0.12357296049594879, "median": 0.12363500148057938, "p05": 0.12287977337837219, "p95": 0.1240825355052948 }, { "label": "completed_group_3", "mean": 0.12375843524932861, "median": 0.12379264831542969, "p05": 0.1232200488448143, "p95": 0.12428808957338333 }, { "label": "completed_group_4", "mean": 0.12469147890806198, "median": 0.12474392354488373, "p05": 0.1235789805650711, "p95": 0.12562167644500732 }, { "label": "completed_group_5", "mean": 0.12560917437076569, "median": 0.12565192580223083, "p05": 0.12473294138908386, "p95": 0.12630751729011536 }, { "label": "completed_group_6", "mean": 0.12766827642917633, "median": 0.12772443890571594, "p05": 0.12707561254501343, "p95": 0.12822651863098145 }, { "label": "current_group_partial", "mean": 0.12797673046588898, "median": 0.12799106538295746, "p05": 0.1274825632572174, "p95": 0.12843307852745056 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0734808444976807, "group": 7, "latest_source_mass": 0.10855817049741745, "layer": 27, "max_source_mass": 0.14781729876995087, "mixer_index": 53, "normalized_entropy": 0.9971335105783546, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1330580711364746, "median": 0.13408532738685608, "p05": 0.12270413339138031, "p95": 0.14177323877811432 }, { "label": "completed_group_1", "mean": 0.14781729876995087, "median": 0.14838829636573792, "p05": 0.13672368228435516, "p95": 0.15774855017662048 }, { "label": "completed_group_2", "mean": 0.1374434530735016, "median": 0.1373273879289627, "p05": 0.13421031832695007, "p95": 0.140645369887352 }, { "label": "completed_group_3", "mean": 0.12885303795337677, "median": 0.12835927307605743, "p05": 0.12514662742614746, "p95": 0.1329982876777649 }, { "label": "completed_group_4", "mean": 0.11949820816516876, "median": 0.11871285736560822, "p05": 0.11462017893791199, "p95": 0.12620851397514343 }, { "label": "completed_group_5", "mean": 0.11495119333267212, "median": 0.11446745693683624, "p05": 0.11059992760419846, "p95": 0.12061949074268341 }, { "label": "completed_group_6", "mean": 0.10982058197259903, "median": 0.1093798577785492, "p05": 0.1069897711277008, "p95": 0.112794429063797 }, { "label": "current_group_partial", "mean": 0.10855817049741745, "median": 0.10835569351911545, "p05": 0.10530182719230652, "p95": 0.11158856004476547 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.07940673828125, "group": 7, "latest_source_mass": 0.12688271701335907, "layer": 28, "max_source_mass": 0.12688271701335907, "mixer_index": 54, "normalized_entropy": 0.9999832631031514, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12457560002803802, "median": 0.12460930645465851, "p05": 0.12360189855098724, "p95": 0.1253867745399475 }, { "label": "completed_group_1", "mean": 0.12431882321834564, "median": 0.12448525428771973, "p05": 0.12329473346471786, "p95": 0.12526866793632507 }, { "label": "completed_group_2", "mean": 0.12479399144649506, "median": 0.12486443668603897, "p05": 0.12387760728597641, "p95": 0.1253962516784668 }, { "label": "completed_group_3", "mean": 0.12437158077955246, "median": 0.12438835948705673, "p05": 0.1238550916314125, "p95": 0.12478173524141312 }, { "label": "completed_group_4", "mean": 0.12397614121437073, "median": 0.12399939447641373, "p05": 0.12361378967761993, "p95": 0.12437361478805542 }, { "label": "completed_group_5", "mean": 0.12488298863172531, "median": 0.12472546845674515, "p05": 0.12454084306955338, "p95": 0.12553870677947998 }, { "label": "completed_group_6", "mean": 0.12619814276695251, "median": 0.12612387537956238, "p05": 0.12555824220180511, "p95": 0.12703651189804077 }, { "label": "current_group_partial", "mean": 0.12688271701335907, "median": 0.12681353092193604, "p05": 0.1262284815311432, "p95": 0.12773586809635162 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.07299542427063, "group": 7, "latest_source_mass": 0.10567984730005264, "layer": 28, "max_source_mass": 0.14826728403568268, "mixer_index": 55, "normalized_entropy": 0.9969000727935836, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13274571299552917, "median": 0.13371163606643677, "p05": 0.12133019417524338, "p95": 0.14282676577568054 }, { "label": "completed_group_1", "mean": 0.14826728403568268, "median": 0.1491231620311737, "p05": 0.1354067623615265, "p95": 0.15911932289600372 }, { "label": "completed_group_2", "mean": 0.13701491057872772, "median": 0.1368696093559265, "p05": 0.13440042734146118, "p95": 0.14018136262893677 }, { "label": "completed_group_3", "mean": 0.12918588519096375, "median": 0.12887251377105713, "p05": 0.12538790702819824, "p95": 0.13307984173297882 }, { "label": "completed_group_4", "mean": 0.12125477194786072, "median": 0.12069438397884369, "p05": 0.11560913175344467, "p95": 0.12839479744434357 }, { "label": "completed_group_5", "mean": 0.1169566810131073, "median": 0.11671148240566254, "p05": 0.11176177859306335, "p95": 0.1237177848815918 }, { "label": "completed_group_6", "mean": 0.10889489948749542, "median": 0.10865428298711777, "p05": 0.10550769418478012, "p95": 0.11299075931310654 }, { "label": "current_group_partial", "mean": 0.10567984730005264, "median": 0.1053324043750763, "p05": 0.10240665078163147, "p95": 0.10906655341386795 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079298734664917, "group": 8, "latest_source_mass": 0.1289007067680359, "layer": 29, "max_source_mass": 0.1289007067680359, "mixer_index": 56, "normalized_entropy": 0.9999313243425908, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12391228973865509, "median": 0.12379568815231323, "p05": 0.12216563522815704, "p95": 0.1260690689086914 }, { "label": "completed_group_1", "mean": 0.12222036719322205, "median": 0.12197070568799973, "p05": 0.12120269238948822, "p95": 0.12389923632144928 }, { "label": "completed_group_2", "mean": 0.12298265099525452, "median": 0.12302859127521515, "p05": 0.12237658351659775, "p95": 0.12344501912593842 }, { "label": "completed_group_3", "mean": 0.12416766583919525, "median": 0.12414702773094177, "p05": 0.12359493225812912, "p95": 0.1248319149017334 }, { "label": "completed_group_4", "mean": 0.125657856464386, "median": 0.1257546842098236, "p05": 0.1244942918419838, "p95": 0.12650099396705627 }, { "label": "completed_group_5", "mean": 0.12553520500659943, "median": 0.1257142722606659, "p05": 0.12441182136535645, "p95": 0.12623384594917297 }, { "label": "completed_group_6", "mean": 0.126623272895813, "median": 0.1266922652721405, "p05": 0.1259365975856781, "p95": 0.12718230485916138 }, { "label": "completed_group_7", "mean": 0.1289007067680359, "median": 0.12883605062961578, "p05": 0.1284138262271881, "p95": 0.1295773684978485 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.189734935760498, "group": 8, "latest_source_mass": 0.09586615115404129, "layer": 29, "max_source_mass": 0.13466420769691467, "mixer_index": 57, "normalized_entropy": 0.9965913172222015, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1203557699918747, "median": 0.12168776988983154, "p05": 0.10881253331899643, "p95": 0.1301186978816986 }, { "label": "completed_group_1", "mean": 0.13466420769691467, "median": 0.13564801216125488, "p05": 0.1213466078042984, "p95": 0.14536651968955994 }, { "label": "completed_group_2", "mean": 0.12461012601852417, "median": 0.12456075102090836, "p05": 0.1213802695274353, "p95": 0.12872551381587982 }, { "label": "completed_group_3", "mean": 0.11655421555042267, "median": 0.11625538021326065, "p05": 0.11300642788410187, "p95": 0.12002266943454742 }, { "label": "completed_group_4", "mean": 0.10905998945236206, "median": 0.10815867781639099, "p05": 0.10389839112758636, "p95": 0.11589331924915314 }, { "label": "completed_group_5", "mean": 0.10487955808639526, "median": 0.10438832640647888, "p05": 0.09981175512075424, "p95": 0.11163480579853058 }, { "label": "completed_group_6", "mean": 0.09900732338428497, "median": 0.09865365922451019, "p05": 0.09571661055088043, "p95": 0.10329514741897583 }, { "label": "completed_group_7", "mean": 0.0950026735663414, "median": 0.09439435601234436, "p05": 0.09206795692443848, "p95": 0.09867438673973083 }, { "label": "current_group_partial", "mean": 0.09586615115404129, "median": 0.0960603654384613, "p05": 0.09171032160520554, "p95": 0.09956925362348557 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.197068691253662, "group": 8, "latest_source_mass": 0.11299411952495575, "layer": 30, "max_source_mass": 0.11490922421216965, "mixer_index": 58, "normalized_entropy": 0.9999290531863855, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1099812239408493, "median": 0.11007699370384216, "p05": 0.10831384360790253, "p95": 0.11145084351301193 }, { "label": "completed_group_1", "mean": 0.10853178799152374, "median": 0.10820644348859787, "p05": 0.10737297683954239, "p95": 0.11033640801906586 }, { "label": "completed_group_2", "mean": 0.10949578881263733, "median": 0.10946135222911835, "p05": 0.10865518450737, "p95": 0.11031430959701538 }, { "label": "completed_group_3", "mean": 0.11001666635274887, "median": 0.11020137369632721, "p05": 0.10912945121526718, "p95": 0.11061263829469681 }, { "label": "completed_group_4", "mean": 0.1107073426246643, "median": 0.11061955988407135, "p05": 0.11008889228105545, "p95": 0.11154733598232269 }, { "label": "completed_group_5", "mean": 0.11118210107088089, "median": 0.11119748651981354, "p05": 0.11032600700855255, "p95": 0.11185683310031891 }, { "label": "completed_group_6", "mean": 0.11218173801898956, "median": 0.11228874325752258, "p05": 0.11149641126394272, "p95": 0.11289148032665253 }, { "label": "completed_group_7", "mean": 0.11490922421216965, "median": 0.11485384404659271, "p05": 0.11418338119983673, "p95": 0.1157391220331192 }, { "label": "current_group_partial", "mean": 0.11299411952495575, "median": 0.11296258866786957, "p05": 0.11219073832035065, "p95": 0.1138962060213089 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1907877922058105, "group": 8, "latest_source_mass": 0.09503427892923355, "layer": 30, "max_source_mass": 0.13228625059127808, "mixer_index": 59, "normalized_entropy": 0.9970704928404667, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11948459595441818, "median": 0.1209583580493927, "p05": 0.10761230438947678, "p95": 0.12862204015254974 }, { "label": "completed_group_1", "mean": 0.13228625059127808, "median": 0.1333099603652954, "p05": 0.12083890289068222, "p95": 0.1419546753168106 }, { "label": "completed_group_2", "mean": 0.12348943948745728, "median": 0.12335763871669769, "p05": 0.11937301605939865, "p95": 0.12711547315120697 }, { "label": "completed_group_3", "mean": 0.11669500172138214, "median": 0.11618717014789581, "p05": 0.11331704258918762, "p95": 0.12055575847625732 }, { "label": "completed_group_4", "mean": 0.11027230322360992, "median": 0.10946162790060043, "p05": 0.10527626425027847, "p95": 0.11702649295330048 }, { "label": "completed_group_5", "mean": 0.1054498553276062, "median": 0.10493277758359909, "p05": 0.10078853368759155, "p95": 0.11155365407466888 }, { "label": "completed_group_6", "mean": 0.10031206905841827, "median": 0.09992779791355133, "p05": 0.09708504378795624, "p95": 0.1038607582449913 }, { "label": "completed_group_7", "mean": 0.09697622060775757, "median": 0.09662114828824997, "p05": 0.09399978816509247, "p95": 0.09983330965042114 }, { "label": "current_group_partial", "mean": 0.09503427892923355, "median": 0.0953635647892952, "p05": 0.09135627001523972, "p95": 0.09895467758178711 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1971726417541504, "group": 8, "latest_source_mass": 0.11179381608963013, "layer": 31, "max_source_mass": 0.11264219880104065, "mixer_index": 60, "normalized_entropy": 0.9999763630979714, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11100216209888458, "median": 0.11096040159463882, "p05": 0.10992560535669327, "p95": 0.1121426597237587 }, { "label": "completed_group_1", "mean": 0.11012112349271774, "median": 0.11014193296432495, "p05": 0.10946580022573471, "p95": 0.110725536942482 }, { "label": "completed_group_2", "mean": 0.1099993959069252, "median": 0.10994885116815567, "p05": 0.10935711860656738, "p95": 0.11085627228021622 }, { "label": "completed_group_3", "mean": 0.1100020483136177, "median": 0.10996703803539276, "p05": 0.1096094623208046, "p95": 0.11043745279312134 }, { "label": "completed_group_4", "mean": 0.10997181385755539, "median": 0.11000630259513855, "p05": 0.10946355760097504, "p95": 0.11040553450584412 }, { "label": "completed_group_5", "mean": 0.11260797083377838, "median": 0.11259360611438751, "p05": 0.11212003231048584, "p95": 0.11315739899873734 }, { "label": "completed_group_6", "mean": 0.11264219880104065, "median": 0.11264394223690033, "p05": 0.11231723427772522, "p95": 0.11297592520713806 }, { "label": "completed_group_7", "mean": 0.11185946315526962, "median": 0.11186999082565308, "p05": 0.11150093376636505, "p95": 0.11219023168087006 }, { "label": "current_group_partial", "mean": 0.11179381608963013, "median": 0.11179733276367188, "p05": 0.11130586266517639, "p95": 0.11215443909168243 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.190375804901123, "group": 8, "latest_source_mass": 0.09452830255031586, "layer": 31, "max_source_mass": 0.13351142406463623, "mixer_index": 61, "normalized_entropy": 0.9968829893376673, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11983220279216766, "median": 0.12115231156349182, "p05": 0.10900401324033737, "p95": 0.12850891053676605 }, { "label": "completed_group_1", "mean": 0.13351142406463623, "median": 0.1344168484210968, "p05": 0.1215919628739357, "p95": 0.14321453869342804 }, { "label": "completed_group_2", "mean": 0.12386792153120041, "median": 0.123808853328228, "p05": 0.1205662339925766, "p95": 0.12756995856761932 }, { "label": "completed_group_3", "mean": 0.11615028977394104, "median": 0.11574137210845947, "p05": 0.11283998191356659, "p95": 0.11950776726007462 }, { "label": "completed_group_4", "mean": 0.10967587679624557, "median": 0.10882684588432312, "p05": 0.10498950630426407, "p95": 0.11603635549545288 }, { "label": "completed_group_5", "mean": 0.10584976524114609, "median": 0.10532175004482269, "p05": 0.10128854960203171, "p95": 0.11199010908603668 }, { "label": "completed_group_6", "mean": 0.10062196850776672, "median": 0.10019652545452118, "p05": 0.09754259884357452, "p95": 0.10435834527015686 }, { "label": "completed_group_7", "mean": 0.09596226364374161, "median": 0.0954611748456955, "p05": 0.09298955649137497, "p95": 0.09913706034421921 }, { "label": "current_group_partial", "mean": 0.09452830255031586, "median": 0.09470570832490921, "p05": 0.09095818549394608, "p95": 0.09791092574596405 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1969356536865234, "group": 8, "latest_source_mass": 0.11499489098787308, "layer": 32, "max_source_mass": 0.11499489098787308, "mixer_index": 62, "normalized_entropy": 0.9998685051802731, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10960094630718231, "median": 0.10919953882694244, "p05": 0.10751543194055557, "p95": 0.11286121606826782 }, { "label": "completed_group_1", "mean": 0.10724945366382599, "median": 0.1071096658706665, "p05": 0.10547052323818207, "p95": 0.10940680652856827 }, { "label": "completed_group_2", "mean": 0.10892991721630096, "median": 0.10898078978061676, "p05": 0.10812374949455261, "p95": 0.10961546003818512 }, { "label": "completed_group_3", "mean": 0.1096237450838089, "median": 0.1097160205245018, "p05": 0.10855352133512497, "p95": 0.11059606075286865 }, { "label": "completed_group_4", "mean": 0.11032412201166153, "median": 0.11054199188947678, "p05": 0.1088760644197464, "p95": 0.11135599762201309 }, { "label": "completed_group_5", "mean": 0.11112386733293533, "median": 0.11126254498958588, "p05": 0.10977865755558014, "p95": 0.11215295642614365 }, { "label": "completed_group_6", "mean": 0.11386257410049438, "median": 0.11392036080360413, "p05": 0.11291038990020752, "p95": 0.11467544734477997 }, { "label": "completed_group_7", "mean": 0.11429048329591751, "median": 0.11428040266036987, "p05": 0.11370178312063217, "p95": 0.11495206505060196 }, { "label": "current_group_partial", "mean": 0.11499489098787308, "median": 0.11495819687843323, "p05": 0.11425958573818207, "p95": 0.1156669408082962 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.188795566558838, "group": 8, "latest_source_mass": 0.09158481657505035, "layer": 32, "max_source_mass": 0.13515985012054443, "mixer_index": 63, "normalized_entropy": 0.9961637918743834, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12045153975486755, "median": 0.12207483500242233, "p05": 0.10754543542861938, "p95": 0.13057786226272583 }, { "label": "completed_group_1", "mean": 0.13515985012054443, "median": 0.13634194433689117, "p05": 0.12046528607606888, "p95": 0.14655664563179016 }, { "label": "completed_group_2", "mean": 0.12505561113357544, "median": 0.12510301172733307, "p05": 0.12129884958267212, "p95": 0.12965641915798187 }, { "label": "completed_group_3", "mean": 0.11722971498966217, "median": 0.11687646061182022, "p05": 0.11354094743728638, "p95": 0.12092718482017517 }, { "label": "completed_group_4", "mean": 0.11039334535598755, "median": 0.10915638506412506, "p05": 0.1051320806145668, "p95": 0.11790860444307327 }, { "label": "completed_group_5", "mean": 0.10568177700042725, "median": 0.10491170734167099, "p05": 0.10036594420671463, "p95": 0.11337733268737793 }, { "label": "completed_group_6", "mean": 0.09951679408550262, "median": 0.09897811710834503, "p05": 0.09581314772367477, "p95": 0.10455189645290375 }, { "label": "completed_group_7", "mean": 0.09492656588554382, "median": 0.09424832463264465, "p05": 0.09176740050315857, "p95": 0.09912464767694473 }, { "label": "current_group_partial", "mean": 0.09158481657505035, "median": 0.09171414375305176, "p05": 0.087994284927845, "p95": 0.09507401287555695 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1922647953033447, "group": null, "latest_source_mass": 0.12173722684383392, "layer": null, "max_source_mass": 0.12246126681566238, "mixer_index": null, "normalized_entropy": 0.9977427059190792, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09991010278463364, "median": 0.09913325309753418, "p05": 0.09100741893053055, "p95": 0.1105685830116272 }, { "label": "completed_group_1", "mean": 0.0929517075419426, "median": 0.09252890944480896, "p05": 0.08571945130825043, "p95": 0.09959838539361954 }, { "label": "completed_group_2", "mean": 0.10029369592666626, "median": 0.10029015690088272, "p05": 0.0971769392490387, "p95": 0.10383395105600357 }, { "label": "completed_group_3", "mean": 0.10801893472671509, "median": 0.10761593282222748, "p05": 0.10540572553873062, "p95": 0.1119115948677063 }, { "label": "completed_group_4", "mean": 0.11530699580907822, "median": 0.11600182950496674, "p05": 0.10986640304327011, "p95": 0.11992081254720688 }, { "label": "completed_group_5", "mean": 0.11817377060651779, "median": 0.11768905073404312, "p05": 0.11345858126878738, "p95": 0.1227451041340828 }, { "label": "completed_group_6", "mean": 0.12114628404378891, "median": 0.12136594951152802, "p05": 0.11863197386264801, "p95": 0.12347076088190079 }, { "label": "completed_group_7", "mean": 0.12246126681566238, "median": 0.12239877134561539, "p05": 0.12033852934837341, "p95": 0.12512150406837463 }, { "label": "completed_group_8", "mean": 0.12173722684383392, "median": 0.12140394747257233, "p05": 0.11953023076057434, "p95": 0.12511639297008514 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "0e9274e8b27f225d1a13bd767f85857d08623357c1eea15d895fa78720bf942c", "optimizer_state_before": "0e9274e8b27f225d1a13bd767f85857d08623357c1eea15d895fa78720bf942c", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5f9ceb4daf4a04de3e80e18a04e2c787acd0577d8e42d246715f712ea2c503dc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.002723898272961378, 0.0025367254856973886, 0.0027546093333512545, 0.002648916095495224, 0.0027809101156890392, 0.0025255572982132435, 0.003397067543119192, 0.0033452848438173532, 0.003559867851436138, 0.0038245979230850935, 0.003386941971257329, 0.00461281044408679, 0.004887574352324009, 0.005150536075234413, 0.0054018739610910416, 0.006125702988356352, 0.005625482182949781, 0.006412550341337919, 0.0072620403952896595, 0.008774910122156143, 0.008495661430060863, 0.007638709619641304, 0.009623398073017597, 0.008996927179396152, 0.00961991585791111, 0.011907265521585941, 0.014190610498189926, 0.013635334558784962, 0.014384749345481396, 0.01446358859539032, 0.013604369945824146, 0.015084062702953815 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.492743683684239e-05, "first_to_last_ratio": 3.25801775252393, "imbalance_abs_log_ratio": 1.1811189590673807, "last_quartile_mean": 4.581754296850704e-06, "mean": 8.43460854582645e-06, "normalized": [ 2.149273354999072, 2.053516748208887, 1.936844350800347, 1.827189890602358, 1.6952544178724434, 1.5885589988048698, 1.4979335388418822, 1.4097022043150367, 1.2877506109767536, 1.206962383078768, 1.1247429419702293, 1.0539031177503007, 0.9733757280016656, 0.9100554611124945, 0.8555710730539545, 0.808590391475426, 0.7636731240457068, 0.7224612895073799, 0.6861986404994214, 0.6566175563965578, 0.639518909427233, 0.6187999065817441, 0.6011551600640996, 0.5866791280052756, 0.5802231951082482, 0.5655891936865475, 0.5545366219021041, 0.5449216791487539, 0.534973976358655, 0.5278906409101619, 0.5216937669853605, 0.515841999508263 ], "peak_layer": 1, "peak_normalized": 2.149273354999072, "population_cv": 0.5023641496635137, "rest_mean": 9.051154595201499e-06, "spike_contrast": 0.5640451530800011, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.105259879201185e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8128279407392256e-05, 1.732060991344042e-05, 1.6336523913196288e-05, 1.5411631466122344e-05, 1.4298807400336955e-05, 1.3398873306869064e-05, 1.2634483027795795e-05, 1.1890286259585992e-05, 1.0861672308237758e-05, 1.0180255230807234e-05, 9.486766430200078e-06, 8.889260243449826e-06, 8.21004323370289e-06, 7.675961569475476e-06, 7.216407084342791e-06, 6.820143426011782e-06, 6.4412838582939e-06, 6.093678166507743e-06, 5.787816917290911e-06, 5.5383120525220875e-06, 5.394091658672551e-06, 5.219334980210988e-06, 5.070508450444322e-06, 4.9484087867313065e-06, 4.893955519946758e-06, 4.770523446495645e-06, 4.677299330069218e-06, 4.5962010517541785e-06, 4.512296072789468e-06, 4.452550911082653e-06, 4.400282705319114e-06, 4.350925337348599e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.9674976455426076e-05, "first_to_last_ratio": 3.267906982188047, "imbalance_abs_log_ratio": 1.1841497133699193, "last_quartile_mean": 1.8260916476720013e-05, "mean": 3.383400519396673e-05, "normalized": [ 2.1128677469499655, 2.0289069422167634, 1.9237165333750805, 1.8241483568045902, 1.696610198326043, 1.5957581812315693, 1.5065087922293456, 1.421542312419023, 1.2976047137215567, 1.216584484419669, 1.1353408195559018, 1.064739590499697, 0.982436165819818, 0.9182427418043383, 0.8622268120476726, 0.8145227282026447, 0.7677725464688829, 0.7258005310001772, 0.6882849152330417, 0.657779832149612, 0.6393622043713836, 0.6177095995408469, 0.5992999823253519, 0.5844668394405714, 0.5774961982840346, 0.5625252092056405, 0.5512933787722648, 0.5414231827170685, 0.5312049841613803, 0.5240801531127671, 0.517855677058088, 0.5118876465352097 ], "peak_layer": 1, "peak_normalized": 2.1128677469499655, "population_cv": 0.49917871029499944, "rest_mean": 3.631725222577289e-05, "spike_contrast": 0.5623903233438747, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.042447122221347e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.148677832446992e-05, 6.864604802103713e-05, 6.508703518193215e-05, 6.171824497869238e-05, 5.740311826230027e-05, 5.3990890592103824e-05, 5.0971226301044226e-05, 4.8096469981828704e-05, 4.390316462377086e-05, 4.1161925764754415e-05, 3.841312718577683e-05, 3.602440483518876e-05, 3.323975033708848e-05, 3.1067829695530236e-05, 2.9172586437198333e-05, 2.7558566216612235e-05, 2.597682032501325e-05, 2.4556738935643807e-05, 2.3287435396923684e-05, 2.2255326257436536e-05, 2.163218414352741e-05, 2.0899589799228124e-05, 2.0276718714740127e-05, 1.9774854081333615e-05, 1.953900937223807e-05, 1.9032480850000866e-05, 1.8652463040780276e-05, 1.8318514776183292e-05, 1.7972792193177156e-05, 1.773173062247224e-05, 1.752113166730851e-05, 1.731920929159969e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05292036524042487, "first_to_last_ratio": 3.26790701098967, "imbalance_abs_log_ratio": 1.1841497221833963, "last_quartile_mean": 0.01619396300520748, "mean": 0.030004334141267464, "normalized": [ 2.1128676597267897, 2.0289069034480245, 1.9237164254948962, 1.8241483367966431, 1.6966102148820104, 1.595758204107866, 1.5065087963694095, 1.421542357002479, 1.297604704701885, 1.2165845161095536, 1.135340842386519, 1.0647396547459889, 0.9824361648697959, 0.9182427925601578, 0.8622268653413654, 0.8145227226086523, 0.7677725509880283, 0.7258005535410471, 0.688284911679496, 0.6577798121305034, 0.6393622162716224, 0.6177096110666057, 0.5993000234249083, 0.5844668186664959, 0.577496193281449, 0.5625251726924361, 0.5512933687361069, 0.5414231479385886, 0.5312049732084057, 0.5240801430831734, 0.5178556786782627, 0.5118876634608343 ], "peak_layer": 1, "peak_normalized": 2.1128676597267897, "population_cv": 0.49917869887275884, "rest_mean": 0.03220650239813107, "spike_contrast": 0.5623903313156743, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01811262555420399, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0633951872587204, 0.06087600067257881, 0.05771983042359352, 0.05473235622048378, 0.050905659794807434, 0.0478796623647213, 0.04520179331302643, 0.0426524318754673, 0.03893376514315605, 0.03650280833244324, 0.03406514599919319, 0.031946804374456406, 0.02947734296321869, 0.027551263570785522, 0.025870542973279953, 0.024439211934804916, 0.023036504164338112, 0.021777162328362465, 0.020651530474424362, 0.019736245274543762, 0.01918363757431507, 0.018533965572714806, 0.017981598153710365, 0.01753653772175312, 0.017327388748526573, 0.016878193244338036, 0.016541190445423126, 0.0162450410425663, 0.015938451513648033, 0.015724675729870796, 0.01553791482001543, 0.015358848497271538 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.549642255824438e-06, "first_to_last_ratio": 3.68048280550862, "imbalance_abs_log_ratio": 1.3030439407243277, "last_quartile_mean": 1.2361536505522963e-06, "mean": 2.4386981998247848e-06, "normalized": [ 2.466847232801335, 2.237590961217267, 2.067896996004643, 1.9400809535365267, 1.7430260468181578, 1.597805887165436, 1.487469389543883, 1.3841051490161118, 1.2636941636501926, 1.1797245287558995, 1.1016637980695667, 1.0304914123685331, 0.9505007964227872, 0.8872713444393213, 0.8269839442149156, 0.7815777037800856, 0.7317613226304623, 0.6893227271477795, 0.652141132025827, 0.6215326520683091, 0.6024681473460322, 0.5840809531927201, 0.5656423861818819, 0.5511940730185599, 0.5440686297089605, 0.529931304143241, 0.5180806767693359, 0.508433674599848, 0.4992370473637191, 0.4907378696025909, 0.48494560322542063, 0.4796914931706492 ], "peak_layer": 1, "peak_normalized": 2.466847232801335, "population_cv": 0.5676326111755923, "rest_mean": 2.633120777351939e-06, "spike_contrast": 0.527441161500707, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3888162811781512e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.015895905875368e-06, 5.456809049064759e-06, 5.042976681579603e-06, 4.73127192890388e-06, 4.2507144826231524e-06, 3.896566340699792e-06, 3.6274889225751394e-06, 3.3754147352738073e-06, 3.081768682022812e-06, 2.8769920845661545e-06, 2.6866255211643875e-06, 2.513057552278042e-06, 2.3179845811682753e-06, 2.1637870304402895e-06, 2.016764256040915e-06, 1.9060321392316837e-06, 1.7845450202003121e-06, 1.6810500937936013e-06, 1.5903754047030816e-06, 1.5157305597313098e-06, 1.4692379863845417e-06, 1.424397169103031e-06, 1.3794310689263511e-06, 1.344195993624453e-06, 1.3268191878523794e-06, 1.2923425174449221e-06, 1.2634424138013856e-06, 1.2399162869769498e-06, 1.2174884886917425e-06, 1.1967615591856884e-06, 1.1826359695987776e-06, 1.1698227808665251e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.834020455746213e-05, "first_to_last_ratio": 2.7330556661570378, "imbalance_abs_log_ratio": 1.0054202748896328, "last_quartile_mean": 1.7687237459540484e-05, "mean": 2.944969719465007e-05, "normalized": [ 1.944393539840578, 1.8698217429189048, 1.7778954351065075, 1.6896780508522091, 1.5856589524406433, 1.4966311985306986, 1.4199003100173502, 1.3476205250759918, 1.2499825935748032, 1.1821154937659675, 1.1141820572947714, 1.0564873202395735, 0.996375680392802, 0.9433030151418219, 0.8970944032316949, 0.8560097734780557, 0.8186006518874163, 0.7814267969140296, 0.7477627937456819, 0.7193270337078107, 0.7043434099536711, 0.683052553167036, 0.6644263425171573, 0.6491784221352348, 0.6422634118610044, 0.6262862823209598, 0.6139376570332481, 0.6029150238009798, 0.5913772622967535, 0.5831537397393776, 0.5759145816028887, 0.568883945414377 ], "peak_layer": 1, "peak_normalized": 1.944393539840578, "population_cv": 0.42167498055732977, "rest_mean": 3.125674791388317e-05, "spike_contrast": 0.6299959088848456, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.969162331079133e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.726180097553879e-05, 5.506568413693458e-05, 5.235848220763728e-05, 4.9760506954044104e-05, 4.6697176003362983e-05, 4.4075335608795285e-05, 4.1815634176600724e-05, 3.968701639678329e-05, 3.68116088793613e-05, 3.481294334051199e-05, 3.2812324207043275e-05, 3.111323167104274e-05, 2.9342962079681456e-05, 2.7779988158727065e-05, 2.6419158530188724e-05, 2.520922862458974e-05, 2.410754132142756e-05, 2.3012782548903488e-05, 2.2021387849235907e-05, 2.118396332662087e-05, 2.074270014418289e-05, 2.0115690858801827e-05, 1.9567154595279135e-05, 1.9118107957183383e-05, 1.8914462998509407e-05, 1.8443941371515393e-05, 1.8080278096022084e-05, 1.7755664885044098e-05, 1.741588130244054e-05, 1.7173701053252444e-05, 1.696051003818866e-05, 1.6753459931351244e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.911581188731361e-05, "first_to_last_ratio": 2.1442152520739666, "imbalance_abs_log_ratio": 0.7627736355916307, "last_quartile_mean": 1.824248374759918e-05, "mean": 2.586868231446715e-05, "normalized": [ 1.7871656412737131, 1.7134487296081482, 1.6333736796405236, 1.556430797867884, 1.4684197123663512, 1.3787761642028489, 1.3133890592163948, 1.2457273633361867, 1.1743095829446748, 1.1167737823796342, 1.0549744950561857, 1.006411191932248, 0.9696254431287762, 0.9309287088943796, 0.9010845828506268, 0.8670296369391367, 0.844384842806383, 0.8202754309623025, 0.7995759366031269, 0.778130175607516, 0.7700498510754012, 0.754237267577021, 0.7412791029837059, 0.7326331543280178, 0.7310833132902649, 0.7218522629025708, 0.7147141764679059, 0.7081022667260183, 0.6994302307319604, 0.6934167828145053, 0.6888024881554871, 0.684164145330101 ], "peak_layer": 1, "peak_normalized": 1.7871656412737131, "population_cv": 0.3324134582891191, "rest_mean": 2.7086155737547585e-05, "spike_contrast": 0.7123316433969433, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9294325829832816e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.623162021744065e-05, 4.432466084836051e-05, 4.2253224819432944e-05, 4.0262813854496926e-05, 3.798608304350637e-05, 3.566712257452309e-05, 3.39756443281658e-05, 3.2225325412582606e-05, 3.0377841540030204e-05, 2.888946619350463e-05, 2.7290800062473863e-05, 2.603453140181955e-05, 2.5082932552322745e-05, 2.4081899027805775e-05, 2.3309870812227018e-05, 2.242891423520632e-05, 2.1843123249709606e-05, 2.1219444533926435e-05, 2.0683975890278816e-05, 2.0129202312091365e-05, 1.9920174963772297e-05, 1.9511124264681712e-05, 1.9175913621438667e-05, 1.8952254322357476e-05, 1.891216197691392e-05, 1.8673366867005825e-05, 1.848871397669427e-05, 1.831767258408945e-05, 1.8093338439939544e-05, 1.7937778466148302e-05, 1.7818412743508816e-05, 1.7698424926493317e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011420926875871373, "first_to_last_ratio": 4.906198539252964, "imbalance_abs_log_ratio": 1.590499413678891, "last_quartile_mean": 2.3278566459339345e-05, "mean": 5.893888680930104e-05, "normalized": [ 2.34070421869906, 2.2660705441771873, 2.12832569527821, 2.0019442205229447, 1.8447664945236562, 1.7382507097156406, 1.640255140843167, 1.5417428136771152, 1.39214202838135, 1.3090459121430928, 1.2137431403699552, 1.1419557102859084, 1.0196179370749445, 0.9468482835406217, 0.8718957397707048, 0.8113961329283941, 0.7366247508392765, 0.6792650123242505, 0.625986195283176, 0.5827208972589306, 0.5455979369449757, 0.513063459434176, 0.4857088480531883, 0.4626395282739635, 0.44411249043994555, 0.42204051663631037, 0.40670988685687265, 0.39333811790230955, 0.3823094762270729, 0.3761622622739262, 0.3695001417649948, 0.36551575755467886 ], "peak_layer": 1, "peak_normalized": 2.34070421869906, "population_cv": 0.6231669472526075, "rest_mean": 6.450288741756662e-05, "spike_contrast": 0.4479378316450088, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8893283524666912e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013795850099995732, 0.00013355967530515045, 0.00012544114724732935, 0.00011799236381193623, 0.00010872848361032084, 0.00010245056182611734, 9.667481208452955e-05, 9.086860518436879e-05, 8.205130143323913e-05, 7.715370884397998e-05, 7.153666956583038e-05, 6.730559834977612e-05, 6.0095146181993186e-05, 5.5806183809181675e-05, 5.1388564315857366e-05, 4.7822784836171195e-05, 4.34158428106457e-05, 4.003512367489748e-05, 3.689492950798012e-05, 3.434492100495845e-05, 3.215693504898809e-05, 3.023938916157931e-05, 2.8627138817682862e-05, 2.7267458790447563e-05, 2.617549580463674e-05, 2.4874598238966428e-05, 2.397102798568085e-05, 2.3182910808827728e-05, 2.2532894945470616e-05, 2.2170584998093545e-05, 2.1777927031507716e-05, 2.154309186153114e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "78bdb1dc451fca8b12adc84e1182b5c9328709671c2cc7151ff16c36d6a3b757", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.004827107302844524, 0.004943604115396738, 0.00549353240057826, 0.005891259294003248, 0.0059933834709227085, 0.006044833920896053, 0.0061737713403999805, 0.0059272525832057, 0.0063678608275949955, 0.0061998614110052586, 0.006479958072304726, 0.006899545900523663, 0.007097913417965174, 0.006852242164313793, 0.006816149223595858, 0.007103090640157461, 0.007109561935067177, 0.007626448292285204, 0.007445307914167643, 0.007442969363182783, 0.007471293676644564, 0.007774693425744772, 0.007449399214237928, 0.007351893465965986, 0.008440978825092316, 0.008385274559259415, 0.008216616697609425, 0.007803319953382015, 0.007909612730145454, 0.007761957123875618, 0.008686874993145466, 0.008338849991559982 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4170842405292206e-05, "first_to_last_ratio": 3.1401414420956977, "imbalance_abs_log_ratio": 1.144267844159062, "last_quartile_mean": 4.5128038550501515e-06, "mean": 8.044083415370551e-06, "normalized": [ 2.1698039188085403, 2.0423624648735483, 1.924499098358531, 1.829561680165496, 1.6732631766752304, 1.5731330769781084, 1.4808173823619837, 1.39974215233296, 1.2679140487801928, 1.180926373046176, 1.1065316311690123, 1.0359566369769586, 0.9549370343414074, 0.8983380270359873, 0.8480121389826683, 0.7997697342488901, 0.7579715143085217, 0.7199721252488075, 0.6886613084032159, 0.6633099718066915, 0.648790261068007, 0.6305361658191545, 0.6153213953215836, 0.6017960242412196, 0.5932141420323949, 0.5815513841228961, 0.5714478326558409, 0.56366655527055, 0.5536526068712324, 0.5470993720351313, 0.540939542718517, 0.5365012229405448 ], "peak_layer": 1, "peak_normalized": 2.1698039188085403, "population_cv": 0.494390404654856, "rest_mean": 8.613229729908872e-06, "spike_contrast": 0.5770998188522956, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.970693316863617e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.745408371789381e-05, 1.6428934031864628e-05, 1.548083128000144e-05, 1.4717146768816747e-05, 1.3459868569043465e-05, 1.2654413694690447e-05, 1.1911818546650466e-05, 1.1259642633376643e-05, 1.0199206371908076e-05, 9.499470252194442e-06, 8.901032742869575e-06, 8.333321602549404e-06, 7.681593160668854e-06, 7.226306024676887e-06, 6.821480383223388e-06, 6.43341445538681e-06, 6.0971860875724815e-06, 5.791515832243022e-06, 5.539649009733694e-06, 5.335720743460115e-06, 5.218922979111085e-06, 5.072085514257196e-06, 4.949696631229017e-06, 4.840897418034729e-06, 4.771864041686058e-06, 4.678047844208777e-06, 4.5967740334162954e-06, 4.534180789050879e-06, 4.4536277528095525e-06, 4.400912985147443e-06, 4.351362804300152e-06, 4.3156605897820555e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.672785118804313e-05, "first_to_last_ratio": 3.1553631949775576, "imbalance_abs_log_ratio": 1.149103606634747, "last_quartile_mean": 1.797823188098846e-05, "mean": 3.227320695486924e-05, "normalized": [ 2.1337751341856186, 2.022538217328439, 1.9170288286790025, 1.8286007981160775, 1.6766598792294076, 1.5813157540855372, 1.4920540136500582, 1.4099335912034971, 1.276811677467349, 1.1911592605052752, 1.116819754454456, 1.0462850360359126, 0.9632481639863413, 0.9048770313780828, 0.8541222938622952, 0.8042705353803223, 0.7611962767945593, 0.7217531538590994, 0.6897404870714406, 0.6635480643338956, 0.6476672777722129, 0.6283660375169337, 0.6128277641309886, 0.5988914175972425, 0.5898193555984405, 0.5780461908459694, 0.5676768961249729, 0.5596908776317308, 0.5494806934414415, 0.5429427329556125, 0.5366812854926843, 0.532171519285104 ], "peak_layer": 1, "peak_normalized": 2.1337751341856186, "population_cv": 0.49221664771975665, "rest_mean": 3.457109293473574e-05, "spike_contrast": 0.5746021018511365, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9864622663590127e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.886376650072634e-05, 6.527379446197301e-05, 6.186866812640801e-05, 5.901481199543923e-05, 5.411119127529673e-05, 5.103413059259765e-05, 4.815336797037162e-05, 4.5503078581532463e-05, 4.1206807509297505e-05, 3.8442529330495745e-05, 3.60433550667949e-05, 3.3766973501769826e-05, 3.1087107345229015e-05, 2.9203283702372573e-05, 2.7565265554585494e-05, 2.5956389436032623e-05, 2.4566244974266738e-05, 2.3293288904824294e-05, 2.2260137484408915e-05, 2.14148240047507e-05, 2.0902300093439408e-05, 2.027938717219513e-05, 1.9777917259489186e-05, 1.9328146663610823e-05, 1.9035362129216082e-05, 1.865540434664581e-05, 1.8320753952139057e-05, 1.806301952456124e-05, 1.77335041371407e-05, 1.7522503185318783e-05, 1.7320426195510663e-05, 1.7174881577375345e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.050306825432926416, "first_to_last_ratio": 3.1553631873240913, "imbalance_abs_log_ratio": 1.1491036042092049, "last_quartile_mean": 0.01594327576458454, "mean": 0.02862020255997777, "normalized": [ 2.1337750803890514, 2.022538074577747, 1.9170287596851208, 1.8286008602762933, 1.6766599442378622, 1.5813157057540668, 1.4920541125825328, 1.40993364991894, 1.2768116947254893, 1.1911592601257468, 1.1168198090844537, 1.0462850213817882, 0.9632482114755018, 0.9048770222788713, 0.8541222536408491, 0.8042704905420582, 0.7611962477207449, 0.7217531610115661, 0.6897404919902839, 0.663548124834154, 0.647667267417541, 0.6283660528035011, 0.6128277842992133, 0.5988913662696712, 0.5898193333225623, 0.578046223698357, 0.5676769216043295, 0.5596908382375889, 0.5494807254443873, 0.5429427375551722, 0.5366812808757728, 0.5321714922387812 ], "peak_layer": 1, "peak_normalized": 2.1337750803890514, "population_cv": 0.49221664059540937, "rest_mean": 0.03065799087010048, "spike_contrast": 0.5746020918316423, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01761614568531513, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.061069075018167496, 0.05788544937968254, 0.054865751415491104, 0.05233492702245712, 0.04798634722828865, 0.0452575758099556, 0.04270289093255997, 0.04035258665680885, 0.036542609333992004, 0.034091219305992126, 0.03196360915899277, 0.02994488924741745, 0.027568358927965164, 0.025897763669490814, 0.024445151910185814, 0.023018384352326393, 0.021785590797662735, 0.02065672166645527, 0.01974051259458065, 0.018990881741046906, 0.018536368384957314, 0.01798396371304989, 0.01753925532102585, 0.01714039221405983, 0.01688074879348278, 0.0165438000112772, 0.01624702848494053, 0.016018465161323547, 0.015726249665021896, 0.015539131127297878, 0.015359926968812943, 0.015230855904519558 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.399239884378403e-06, "first_to_last_ratio": 3.612110266791686, "imbalance_abs_log_ratio": 1.284292162993447, "last_quartile_mean": 1.2179140611578987e-06, "mean": 2.349132376622265e-06, "normalized": [ 2.4955076017050923, 2.2436808131458656, 2.0874700594685764, 1.9621517331257625, 1.7419934380554116, 1.5859298059239806, 1.4811826529402687, 1.383750925243364, 1.24754299116381, 1.1572103620070064, 1.0826808130566368, 1.0127662347026543, 0.9261323573573094, 0.8680208953912559, 0.8137959587886114, 0.7661051062366301, 0.7198218777908413, 0.6807070982717376, 0.6465769322329894, 0.6214722858706658, 0.6055471528554645, 0.5897479442686585, 0.5729654410868715, 0.5596177904436604, 0.5505791556557464, 0.5392724776945974, 0.5286699021580167, 0.5214282796203655, 0.5112551634910032, 0.5041753262585503, 0.49807684781126477, 0.49416457617733067 ], "peak_layer": 1, "peak_normalized": 2.4955076017050923, "population_cv": 0.5694378379798026, "rest_mean": 2.5337169770662313e-06, "spike_contrast": 0.5337516172744564, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3523755342248478e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.862277703272412e-06, 5.270703240967123e-06, 4.903743501927238e-06, 4.6093541641312186e-06, 4.092173185199499e-06, 3.725559054146288e-06, 3.479494125713245e-06, 3.2506140996702015e-06, 2.9306436317710904e-06, 2.718440327953431e-06, 2.543360551499063e-06, 2.3791219518898288e-06, 2.175607505705557e-06, 2.0390959889482474e-06, 1.9117144347546855e-06, 1.7996823089561076e-06, 1.6909568785195006e-06, 1.5990710835467326e-06, 1.5188948054856155e-06, 1.459920667912229e-06, 1.4225104223442031e-06, 1.385395989927929e-06, 1.3459716683428269e-06, 1.3146162700650166e-06, 1.2933833204442635e-06, 1.266822437173687e-06, 1.2419155837051221e-06, 1.224904053742648e-06, 1.201006057272025e-06, 1.1843745824080543e-06, 1.1700484492394025e-06, 1.160858005277987e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.614919907908188e-05, "first_to_last_ratio": 2.6506059611493034, "imbalance_abs_log_ratio": 0.9747882784430122, "last_quartile_mean": 1.7410810869478155e-05, "mean": 2.826825851798276e-05, "normalized": [ 1.9567333803186102, 1.8597972147876323, 1.766445715132986, 1.6894057707724273, 1.5636321818586363, 1.4819572930902338, 1.4065416062160392, 1.3358454162549211, 1.2332908204028759, 1.1620147315245914, 1.1014502996752689, 1.0446856471155146, 0.9835188591006802, 0.9354808101194146, 0.8921368408217227, 0.8494656881105699, 0.8144325093861817, 0.7792466985899428, 0.7495668388307936, 0.7246367192704585, 0.7117068137537608, 0.6923040729201976, 0.6764241600957448, 0.6619694138511188, 0.6525606633107036, 0.6397070094825148, 0.6281961575410799, 0.6192140303718157, 0.607592629568897, 0.6000373420122876, 0.5927022513148152, 0.5873004143975631 ], "peak_layer": 1, "peak_normalized": 1.9567333803186102, "population_cv": 0.41380193863129383, "rest_mean": 2.9948685955282093e-05, "spike_contrast": 0.6408945749815472, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.919395035656635e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.531344504561275e-05, 5.25732284586411e-05, 4.9934344133362174e-05, 4.77565590699669e-05, 4.420115874381736e-05, 4.1892351873684675e-05, 3.97604817408137e-05, 3.77620235667564e-05, 3.486298373900354e-05, 3.284813283244148e-05, 3.1136081815930083e-05, 2.9531443942687474e-05, 2.7802365366369486e-05, 2.6444413379067555e-05, 2.521915484976489e-05, 2.4012915673665702e-05, 2.3022588720778003e-05, 2.2027947125025094e-05, 2.118894917657599e-05, 2.0484218111960217e-05, 2.0118712200201117e-05, 1.957023050636053e-05, 1.912133302539587e-05, 1.8712722521740943e-05, 1.8446753529133275e-05, 1.8083403119817376e-05, 1.7758011381374672e-05, 1.7504102288512513e-05, 1.7175585526274517e-05, 1.6962010704446584e-05, 1.6754660464357585e-05, 1.6601959941908717e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.735190557563328e-05, "first_to_last_ratio": 2.064507245129547, "imbalance_abs_log_ratio": 0.7248915757248915, "last_quartile_mean": 1.8092407117364928e-05, "mean": 2.5017036136887327e-05, "normalized": [ 1.7806348537056282, 1.6958575783596996, 1.6145104713051759, 1.5397532195284758, 1.4341567363816627, 1.3605904748672402, 1.2904091560066684, 1.2285577888193309, 1.1575565189704986, 1.0915619889550103, 1.0406567529663657, 0.9956018385584979, 0.9640474313507927, 0.9320995174703498, 0.8974595875050125, 0.8691926915133404, 0.8490635018802836, 0.8275552938519668, 0.8049190624458126, 0.7866702995626351, 0.780171913500315, 0.7668318773324383, 0.7578168517732956, 0.7486969056434284, 0.7465158956368789, 0.7394522618151603, 0.7324985655561136, 0.7264758484984223, 0.7170356146942368, 0.712241115433273, 0.7073157381215093, 0.7040926479904809 ], "peak_layer": 1, "peak_normalized": 1.7806348537056282, "population_cv": 0.31924035304186044, "rest_mean": 2.6128873421974412e-05, "spike_contrast": 0.7276668415955134, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9013114797417075e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.454620648175478e-05, 4.242533032083884e-05, 4.039026680402458e-05, 3.8520061934832484e-05, 3.587835090002045e-05, 3.403794107725844e-05, 3.22822124871891e-05, 3.073487459914759e-05, 2.8958633265574463e-05, 2.73076457233401e-05, 2.60341475950554e-05, 2.4907007173169404e-05, 2.4117609427776188e-05, 2.331836731173098e-05, 2.2451778932008892e-05, 2.1744624973507598e-05, 2.1241052309051156e-05, 2.0702980691567063e-05, 2.013668927247636e-05, 1.968015931197442e-05, 1.9517588953021914e-05, 1.918386078614276e-05, 1.8958331565954722e-05, 1.873017754405737e-05, 1.8675615137908608e-05, 1.8498903955332935e-05, 1.8324943084735423e-05, 1.8174272554460913e-05, 1.793810588424094e-05, 1.7818161722971126e-05, 1.769494338077493e-05, 1.761431121849455e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010834934892045567, "first_to_last_ratio": 4.772634589412918, "imbalance_abs_log_ratio": 1.5628984772495462, "last_quartile_mean": 2.270220920763677e-05, "mean": 5.5645747750077135e-05, "normalized": [ 2.4035622873585307, 2.266667857217833, 2.121353069055484, 2.022827767132241, 1.842912339502741, 1.7402381826456954, 1.6341450450508916, 1.545311039376737, 1.3884783081507637, 1.284206979436914, 1.2074118445936788, 1.1168610262727916, 1.0022982360357862, 0.9246149481154046, 0.8572930756226724, 0.7854058282657367, 0.7196284333802556, 0.6628520139237241, 0.6164320412062984, 0.5776386785157045, 0.5436680818313624, 0.5148121289164611, 0.490108136365702, 0.46745320481928015, 0.44716915492660686, 0.43065012877775255, 0.41644951885427656, 0.4076017882404048, 0.3988567002714443, 0.391504708352861, 0.3871572366322811, 0.3844302111516827 ], "peak_layer": 1, "peak_normalized": 2.4035622873585307, "population_cv": 0.6252223005215759, "rest_mean": 6.087395282789927e-05, "spike_contrast": 0.4503312017101952, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.741344032983761e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001337480207439512, 0.0001261304278159514, 0.00011804427776951343, 0.00011256176367169246, 0.00010255023516947404, 9.683685493655503e-05, 9.093322296394035e-05, 8.598998829256743e-05, 7.726291369181126e-05, 7.1460657636635e-05, 6.718733493471518e-05, 6.214856693986803e-05, 5.5773634812794626e-05, 5.145089016878046e-05, 4.770471423398703e-05, 4.3704494601115584e-05, 4.004426227766089e-05, 3.6884895962430164e-05, 3.4301821870030835e-05, 3.214313619537279e-05, 3.0252816941356286e-05, 2.8647105864365585e-05, 2.727243372646626e-05, 2.6011783120338805e-05, 2.4883061996661127e-05, 2.3963848434505053e-05, 2.3173644876806065e-05, 2.2681306290905923e-05, 2.2194679331732914e-05, 2.178557224397082e-05, 2.1543653929256834e-05, 2.1391906557255425e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ae4066fc37f2b8857b68206f2c50d90301cff538a7afaed7e0b4959067354e04", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.002723898272961378, 0.006246842443943024, 0.009986688382923603, 0.014312135986983776, 0.0027809101156890392, 0.007167908828705549, 0.012327533215284348, 0.017696699127554893, 0.003559867851436138, 0.008642485365271568, 0.014099182561039925, 0.020258057862520218, 0.004887574352324009, 0.011225050315260887, 0.01857108250260353, 0.026577936485409737, 0.005625482182949781, 0.013686468824744225, 0.023031942546367645, 0.03310495615005493, 0.008495661430060863, 0.017283262684941292, 0.029314789921045303, 0.04071469604969025, 0.00961991585791111, 0.023407738655805588, 0.0400763601064682, 0.05648425966501236, 0.014384749345481396, 0.03149676322937012, 0.04818879812955856, 0.06738673150539398 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4927425922905968e-05, "first_to_last_ratio": 3.2580194125343316, "imbalance_abs_log_ratio": 1.181119468582707, "last_quartile_mean": 4.581748612508818e-06, "mean": 8.434603628870718e-06, "normalized": [ 2.149249807253593, 2.0535339039963048, 1.9368405197515186, 1.8272069144524385, 1.6952778345494546, 1.5885491419580562, 1.4979415287755153, 1.4096717556974747, 1.2877403631156048, 1.2069779670771674, 1.1247537335636728, 1.0538960762660219, 0.9733880487902208, 0.9100528645893027, 0.8555679056241683, 0.8085854713945597, 0.7636821416331455, 0.7224543782958864, 0.6861960752222711, 0.6566025735424599, 0.6395340008907516, 0.6187919105654988, 0.6011578827453019, 0.5866789847797121, 0.5802252586132757, 0.5655851563233361, 0.5545362442810942, 0.5449205411198527, 0.5349741264784901, 0.5278853954517888, 0.5216927232447175, 0.5158487699573434 ], "peak_layer": 1, "peak_normalized": 2.149249807253593, "population_cv": 0.5023647441278015, "rest_mean": 9.05114565183693e-06, "spike_contrast": 0.5640475693612389, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.105276704853167e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8128070223610848e-05, 1.7320744518656284e-05, 1.6336482076440006e-05, 1.5411766071338207e-05, 1.4298996575234924e-05, 1.3398782357398886e-05, 1.2634543054446112e-05, 1.1890022506122477e-05, 1.0861579539778177e-05, 1.0180380741076078e-05, 9.486851922702044e-06, 8.889195669326e-06, 8.210142368625384e-06, 7.675935194129124e-06, 7.2163761615229305e-06, 6.8200979512766935e-06, 6.441356163122691e-06, 6.093616320868023e-06, 5.787791906186612e-06, 5.538182449527085e-06, 5.394215804699343e-06, 5.2192644943716004e-06, 5.070528459327761e-06, 4.9484046940051485e-06, 4.893970071861986e-06, 4.770486611960223e-06, 4.677293418353656e-06, 4.5961887735757045e-06, 4.512294708547415e-06, 4.452504072105512e-06, 4.400271336635342e-06, 4.350979907030705e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.967510560367373e-05, "first_to_last_ratio": 3.2679096193643358, "imbalance_abs_log_ratio": 1.184150520362052, "last_quartile_mean": 1.8260941260450636e-05, "mean": 3.383405919521465e-05, "normalized": [ 2.1128501814933904, 2.028923058304855, 1.9237119576664479, 1.8241641545550669, 1.696600823933448, 1.5957782143766441, 1.5065145595844665, 1.4215241299739685, 1.2975903849100163, 1.2165984562514365, 1.1353482545596325, 1.0647386437786797, 0.9824378235158376, 0.9182368677416404, 0.8622246832113106, 0.8145243313263993, 0.7677753532124987, 0.7257958780424952, 0.688282795208288, 0.6577753415192462, 0.6393623666755853, 0.6177067857274526, 0.5993015526233388, 0.5844680033165289, 0.5774972657614055, 0.5625215157524979, 0.5512919612531849, 0.5414258131074452, 0.5312052115670245, 0.5240772736894307, 0.5178563021046121, 0.5118900552557252 ], "peak_layer": 1, "peak_normalized": 2.1128501814933904, "population_cv": 0.49917932069280174, "rest_mean": 3.631730874918436e-05, "spike_contrast": 0.5623905599623192, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0424511603778228e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.148629811126739e-05, 6.864670285722241e-05, 6.508698425022885e-05, 6.171887798700482e-05, 5.740289270761423e-05, 5.399165456765331e-05, 5.0971502787433565e-05, 4.8095931560965255e-05, 4.3902749894186854e-05, 4.1162464185617864e-05, 3.841344005195424e-05, 3.602443030104041e-05, 3.323985947645269e-05, 3.1067680538399145e-05, 2.9172560971346684e-05, 2.7558664442040026e-05, 2.5976956749218516e-05, 2.4556620701332577e-05, 2.3287400836125016e-05, 2.225520984211471e-05, 2.163222416129429e-05, 2.0899527953588404e-05, 2.0276804207242094e-05, 1.9774925021920353e-05, 1.9539076674846e-05, 1.903238626255188e-05, 1.865244485088624e-05, 1.8318633010494523e-05, 1.7972828572965227e-05, 1.7731661500874907e-05, 1.7521180780022405e-05, 1.7319318430963904e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05292047979310155, "first_to_last_ratio": 3.2679095977338717, "imbalance_abs_log_ratio": 1.184150513743, "last_quartile_mean": 0.016193985240533948, "mean": 0.030004381958860904, "normalized": [ 2.112850138441349, 2.0289230386859303, 1.9237119939527503, 1.8241640534063661, 1.6966008064731326, 1.5957782577626325, 1.50651458990766, 1.4215240751000642, 1.2975903450635375, 1.216598469505261, 1.1353482207159766, 1.0647387028334923, 0.9824378272874627, 0.9182369215521707, 0.8622246841932674, 0.8145243422322688, 0.7677753625406581, 0.7257958583332109, 0.688282821503116, 0.6577753494846466, 0.6393623768313474, 0.6177068263368124, 0.5993015514924139, 0.5844679978988628, 0.577497259464603, 0.562521544722804, 0.5512919314350962, 0.541425823588487, 0.5312051819787147, 0.5240772592530296, 0.5178563122373452, 0.51189007578553 ], "peak_layer": 1, "peak_normalized": 2.112850138441349, "population_cv": 0.4991793114170283, "rest_mean": 0.0322065524480961, "spike_contrast": 0.5623905677635356, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.018112661316990852, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06339476257562637, 0.06087658181786537, 0.05771978944540024, 0.05473291501402855, 0.05090545862913132, 0.04788034036755562, 0.04520203918218613, 0.0426519513130188, 0.038933396339416504, 0.03650328516960144, 0.03406542167067528, 0.031946826726198196, 0.02947743982076645, 0.027551131322979927, 0.025870518758893013, 0.02443929947912693, 0.023036625236272812, 0.02177705615758896, 0.020651500672101974, 0.019736142829060555, 0.019183672964572906, 0.018533911556005478, 0.017981672659516335, 0.017536601051688194, 0.01732744835317135, 0.01687811128795147, 0.016541173681616783, 0.016245147213339806, 0.01593848317861557, 0.01572461426258087, 0.015537958592176437, 0.015358945354819298 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.549713025880919e-06, "first_to_last_ratio": 3.680528123890709, "imbalance_abs_log_ratio": 1.3030562538108112, "last_quartile_mean": 1.236157658013326e-06, "mean": 2.4387289627725295e-06, "normalized": [ 2.46688156577926, 2.2376955014199336, 2.0678263447651153, 1.940166497412543, 1.7430074161430915, 1.5977933771333281, 1.4874509058079846, 1.3840448948125312, 1.2636490405882963, 1.179790668094215, 1.1016966117962113, 1.0305101131331023, 0.950494959962906, 0.8872551174370253, 0.827006144436166, 0.781561644601688, 0.7317791765691539, 0.6893042887937005, 0.6521328590808986, 0.6215289607815456, 0.6024501053460986, 0.584043470655544, 0.565626393700991, 0.5512156498280469, 0.5440612538544417, 0.5299479280364501, 0.5180756798945998, 0.5084212008038946, 0.4992530328651117, 0.49069438546998123, 0.48492974295315366, 0.4797050680429897 ], "peak_layer": 1, "peak_normalized": 2.46688156577926, "population_cv": 0.5676426036440108, "rest_mean": 2.6331591698181e-06, "spike_contrast": 0.5274295077355267, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3888058447264485e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.016055522195529e-06, 5.45713282917859e-06, 5.042867996962741e-06, 4.731540229840903e-06, 4.250722668075468e-06, 3.896584985341178e-06, 3.6274896046961658e-06, 3.375310370756779e-06, 3.081697514062398e-06, 2.8771896722901147e-06, 2.6867394353757845e-06, 2.5131348593276925e-06, 2.3179995878308546e-06, 2.1637747522618156e-06, 2.01684383682732e-06, 1.9060170188822667e-06, 1.7846110722530284e-06, 1.6810263332445174e-06, 1.5903752910162439e-06, 1.515740677859867e-06, 1.469212520532892e-06, 1.4243237274058629e-06, 1.379409468427184e-06, 1.3442655699691386e-06, 1.3268179372971645e-06, 1.292399360863783e-06, 1.2634461654670304e-06, 1.239901507688046e-06, 1.2175428310001735e-06, 1.196670609715511e-06, 1.1826122090496938e-06, 1.169870643025206e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.834035416934057e-05, "first_to_last_ratio": 2.733059908807899, "imbalance_abs_log_ratio": 1.0054218272354634, "last_quartile_mean": 1.7687264744381537e-05, "mean": 2.9449753014887392e-05, "normalized": [ 1.9443940544381229, 1.8698233871166146, 1.7778950299739502, 1.6896852248339365, 1.5856582940221604, 1.4966405913970182, 1.4199046599899854, 1.3476142647904932, 1.2499731829995469, 1.1821174532157463, 1.1141837749131207, 1.0564879118975747, 0.9963758300960397, 0.9433002389138315, 0.8970939381604119, 0.8560091392167688, 0.8186039797822264, 0.7814222892392128, 0.7477601410889809, 0.719324496715247, 0.7043446073134325, 0.6830500849305705, 0.66442780083491, 0.6491794152276021, 0.642265097484849, 0.626282254003255, 0.613935752160955, 0.6029174634318201, 0.5913776237573929, 0.5831511520257966, 0.5759147253086592, 0.5688861407197677 ], "peak_layer": 1, "peak_normalized": 1.9443940544381229, "population_cv": 0.4216757923764803, "rest_mean": 3.125680403307477e-05, "spike_contrast": 0.6299965119862723, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9691677516675553e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.726192466681823e-05, 5.5065836932044476e-05, 5.235856951912865e-05, 4.97608125442639e-05, 4.669724512496032e-05, 4.407569576869719e-05, 4.181584154139273e-05, 3.968690725741908e-05, 3.68114015145693e-05, 3.4813067031791434e-05, 3.2812436984386295e-05, 3.111330806859769e-05, 2.9343022106331773e-05, 2.7779959054896608e-05, 2.6419194909976795e-05, 2.5209257728420198e-05, 2.410768502159044e-05, 2.3012693418422714e-05, 2.2021351469447836e-05, 2.1183928765822202e-05, 2.0742774722748436e-05, 2.011565629800316e-05, 1.956723463081289e-05, 1.911817344080191e-05, 1.8914548491011374e-05, 1.844385769800283e-05, 1.8080256268149242e-05, 1.7755770386429504e-05, 1.7415924958186224e-05, 1.717365739750676e-05, 1.696054641797673e-05, 1.6753556337789632e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.911596377292881e-05, "first_to_last_ratio": 2.1441510473750993, "imbalance_abs_log_ratio": 0.7627436919280587, "last_quartile_mean": 1.8243100839754334e-05, "mean": 2.586884403399381e-05, "normalized": [ 1.7871422338212197, 1.7137406573133611, 1.6331955543594734, 1.556344845447187, 1.4684323304243458, 1.3786204440301357, 1.3134601648053343, 1.2457662653653259, 1.1742886707715572, 1.116694094274204, 1.054651337969624, 1.006236775170543, 0.969530361649706, 0.9312138561082258, 0.901034861675357, 0.8672035220710215, 0.8443849784393078, 0.8203617838993202, 0.799640410086616, 0.7782482231951742, 0.770066975636356, 0.7541424075474804, 0.7413032280416163, 0.732574782644356, 0.7311844276122933, 0.7218147017875606, 0.7147312250657709, 0.7081409436201098, 0.6994441403492052, 0.6933838996779177, 0.688757539541391, 0.6842643575989029 ], "peak_layer": 1, "peak_normalized": 1.7871422338212197, "population_cv": 0.33240131193143013, "rest_mean": 2.7086335480648644e-05, "spike_contrast": 0.7123292937076053, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.92943902220577e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.623130371328443e-05, 4.433248977875337e-05, 4.2248881072737277e-05, 4.026084206998348e-05, 3.798664693022147e-05, 3.566331724869087e-05, 3.3977696148213e-05, 3.222653322154656e-05, 3.037749047507532e-05, 2.8887585358461365e-05, 2.72826109721791e-05, 2.603018219815567e-05, 2.5080629711737856e-05, 2.4089426005957648e-05, 2.3308730305870995e-05, 2.2433552658185363e-05, 2.184326331189368e-05, 2.1221811039140448e-05, 2.068577305180952e-05, 2.0132381905568764e-05, 1.9920742488466203e-05, 1.9508792320266366e-05, 1.9176657588104717e-05, 1.8950862795463763e-05, 1.891489591798745e-05, 1.867251194198616e-05, 1.8489270587451756e-05, 1.8318787624593824e-05, 1.8093811377184466e-05, 1.7937039956450462e-05, 1.7817361367633566e-05, 1.7701127944746986e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011420908140280517, "first_to_last_ratio": 4.906424645343274, "imbalance_abs_log_ratio": 1.5905454984181289, "last_quartile_mean": 2.327745551156113e-05, "mean": 5.893761050401736e-05, "normalized": [ 2.3409386034658266, 2.26653318009987, 2.128165373071052, 2.001759557425238, 1.8449035999101382, 1.7381229263770657, 1.6401158530503868, 1.5418310130397193, 1.392320688071593, 1.3090601862719393, 1.213617825327422, 1.1417595841795705, 1.019599771791136, 0.9467828652279454, 0.8718925846836042, 0.8115322793787068, 0.7365786680304162, 0.6791875033995677, 0.6260459221300255, 0.5825401905799077, 0.5456521576869235, 0.5130954641492332, 0.48578325250896687, 0.46257467326905344, 0.4440240870048044, 0.4220562606793032, 0.4066511043445411, 0.39330858167914856, 0.38230571865595697, 0.3762268256398313, 0.369497495638699, 0.36553620323240776 ], "peak_layer": 1, "peak_normalized": 2.3409386034658266, "population_cv": 0.6231971411979633, "rest_mean": 6.45014745343899e-05, "spike_contrast": 0.4479392905134439, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8892744740005583e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013796932762488723, 0.00013358404976315796, 0.00012542898184619844, 0.00011797892511822283, 0.0001087342097889632, 0.00010244081204291433, 9.666450932854787e-05, 9.087183570954949e-05, 8.2060054410249e-05, 7.715287938481197e-05, 7.152773468988016e-05, 6.729258166160434e-05, 6.009277421981096e-05, 5.5801119742682204e-05, 5.1387265557423234e-05, 4.782977339345962e-05, 4.341218664194457e-05, 4.002968853455968e-05, 3.689765071612783e-05, 3.433352685533464e-05, 3.215943434042856e-05, 3.024062061740551e-05, 2.8630904125748202e-05, 2.726304592215456e-05, 2.6169718694291078e-05, 2.4874987502698787e-05, 2.3967044398887083e-05, 2.3180667994893156e-05, 2.2532185539603233e-05, 2.2173910110723227e-05, 2.1777299480163492e-05, 2.1543830371228978e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "0a445d8d97762f2c9a1505daa03132d58d2d7cce48cebe69f58dcee8b68ddc3c", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0055193523876369, 0.009546536952257156, 0.013977172784507275, 0.018744563683867455, 0.00660830270498991, 0.011765030212700367, 0.017156247049570084, 0.022267481312155724, 0.007449587807059288, 0.013175037689507008, 0.019073888659477234, 0.025712018832564354, 0.008972279727458954, 0.01624389924108982, 0.023743286728858948, 0.03214457631111145, 0.00991375558078289, 0.019281653687357903, 0.02868141047656536, 0.03889070823788643, 0.0127727584913373, 0.022841427475214005, 0.034929387271404266, 0.04634999856352806, 0.014845813624560833, 0.029589947313070297, 0.046351123601198196, 0.06254568696022034, 0.01947367936372757, 0.0370744951069355, 0.05507569760084152, 0.07403328269720078 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4170829444992705e-05, "first_to_last_ratio": 3.140130857329134, "imbalance_abs_log_ratio": 1.1442644733604534, "last_quartile_mean": 4.512814939516829e-06, "mean": 8.044093192438595e-06, "normalized": [ 2.1698218591421137, 2.0423579473697657, 1.9244829654882192, 1.829554142460677, 1.6732305026788175, 1.5731474461065056, 1.48081501720716, 1.399743051502537, 1.2679172563879277, 1.1809283296176842, 1.10654046198268, 1.035952438184134, 0.9549549814383372, 0.8983396486930857, 0.8480063030735002, 0.799768196862845, 0.7579745502722912, 0.7199744159526508, 0.6886643720771628, 0.663306678196146, 0.648790207419826, 0.630529124410062, 0.6153188384200302, 0.6018030376590212, 0.5932194699258551, 0.5815579133602139, 0.5714511518587554, 0.5636597082007567, 0.5536559477030776, 0.5471005160904301, 0.5409418248973891, 0.5364916953603418 ], "peak_layer": 1, "peak_normalized": 2.1698218591421137, "population_cv": 0.49438916638334635, "rest_mean": 8.613239397500703e-06, "spike_contrast": 0.577100374865415, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.970703685103217e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.7454249245929532e-05, 1.6428917660959996e-05, 1.5480720321647823e-05, 1.4717104022565763e-05, 1.3459622095979284e-05, 1.2654544661927503e-05, 1.1911813999176957e-05, 1.1259663551754784e-05, 1.019924457068555e-05, 9.499497537035495e-06, 8.901114597392734e-06, 8.333297955687158e-06, 7.681746865273453e-06, 7.22632785254973e-06, 6.821441729698563e-06, 6.433409907913301e-06, 6.0972179198870435e-06, 5.791541298094671e-06, 5.539680387300905e-06, 5.335700734576676e-06, 5.218928890826646e-06, 5.072035037301248e-06, 4.9496820793137886e-06, 4.8409597184218e-06, 4.771912699652603e-06, 4.67810605186969e-06, 4.596806320478208e-06, 4.534131221589632e-06, 4.453660039871465e-06, 4.400927537062671e-06, 4.351386451162398e-06, 4.315589194447966e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.6727842547843466e-05, "first_to_last_ratio": 3.155359202626439, "imbalance_abs_log_ratio": 1.1491023413751247, "last_quartile_mean": 1.79782518898719e-05, "mean": 3.227324236831919e-05, "normalized": [ 2.133789250541258, 2.0225341944070556, 1.9170292050567193, 1.8285814320482991, 1.67664620337743, 1.581315371598383, 1.4920577871848402, 1.409935200362978, 1.2768241415105412, 1.1911541208190792, 1.1168170635514478, 1.0462850151897058, 0.9632588303411923, 0.904875136661487, 0.8541193275958588, 0.8042725273235084, 0.7611969633115008, 0.7217542218299198, 0.6897417592562383, 0.6635457580819046, 0.647670850610065, 0.628363657145899, 0.6128288389014056, 0.5988935221787641, 0.5898175247840765, 0.5780479801280578, 0.5676825294116558, 0.559690432569022, 0.5494813304632692, 0.5429417990112825, 0.5366821620079203, 0.5321658627392354 ], "peak_layer": 1, "peak_normalized": 2.133789250541258, "population_cv": 0.49221598973047537, "rest_mean": 3.45711278323843e-05, "spike_contrast": 0.5746026267549044, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.98646608623676e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.886429764563218e-05, 6.52737362543121e-05, 6.186874816194177e-05, 5.901425174670294e-05, 5.411080928752199e-05, 5.1034174248343334e-05, 4.815354259335436e-05, 4.550318044493906e-05, 4.120725498069078e-05, 3.8442405639216304e-05, 3.604330777307041e-05, 3.37670098815579e-05, 3.108748569502495e-05, 2.9203254598542117e-05, 2.7565200070966966e-05, 2.5956482204492204e-05, 2.4566294086980633e-05, 2.329334893147461e-05, 2.2260202968027443e-05, 2.14147730730474e-05, 2.0902438336634077e-05, 2.0279332602513023e-05, 1.9777973648160696e-05, 1.9328235794091597e-05, 1.9035323930438608e-05, 1.8655482563190162e-05, 1.832095585996285e-05, 1.8063024981529452e-05, 1.7733544154907577e-05, 1.752249227138236e-05, 1.7320473489235155e-05, 1.7174717868329026e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05030681798234582, "first_to_last_ratio": 3.1553591948948423, "imbalance_abs_log_ratio": 1.1491023389248185, "last_quartile_mean": 0.01594329357612878, "mean": 0.028620234137633815, "normalized": [ 2.1337891266389915, 2.022534150927612, 1.9170292478198825, 1.8285814009026793, 1.6766462495068444, 1.5813153928268515, 1.4920578030251115, 1.409935218199867, 1.2768242133981067, 1.1911540409956152, 1.1168171450690334, 1.046284973364676, 0.9632588633452485, 0.9048751127582434, 0.8541193588171093, 0.8042725318270637, 0.7611969698214163, 0.7217542520381491, 0.6897417485003263, 0.6635457656842302, 0.6476708481976686, 0.6283636673897187, 0.6128288002625611, 0.5988935039934771, 0.5898175110899254, 0.5780479939328508, 0.567682478001312, 0.559690415956301, 0.5494813557307923, 0.5429418131013527, 0.5366821530691603, 0.532165893807823 ], "peak_layer": 1, "peak_normalized": 2.1337891266389915, "population_cv": 0.4922159837762602, "rest_mean": 0.030658022017666587, "spike_contrast": 0.5746026138054693, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01761617958545685, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0610695444047451, 0.05788540095090866, 0.05486582592129707, 0.05233442783355713, 0.04798600822687149, 0.04525761678814888, 0.042703043669462204, 0.040352676063776016, 0.03654300794005394, 0.03409110754728317, 0.031963568180799484, 0.029944920912384987, 0.027568694204092026, 0.025897737592458725, 0.024445096030831337, 0.02301846817135811, 0.021785635501146317, 0.020656775683164597, 0.019740570336580276, 0.018990835174918175, 0.018536491319537163, 0.01798391528427601, 0.01753930374979973, 0.017140472307801247, 0.016880715265870094, 0.01654386892914772, 0.016247205436229706, 0.016018470749258995, 0.015726285055279732, 0.015539121814072132, 0.0153599688783288, 0.015230712480843067 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.399315798764292e-06, "first_to_last_ratio": 3.6121892887675737, "imbalance_abs_log_ratio": 1.2843140397097386, "last_quartile_mean": 1.2179084336594315e-06, "mean": 2.3491519840490582e-06, "normalized": [ 2.495529360167265, 2.243705447822421, 2.0875170981243216, 1.9621829763583638, 1.7419047574397617, 1.5859991304051935, 1.4811645795131025, 1.3837971590715752, 1.247616882237959, 1.1571559864040741, 1.082652031260043, 1.012746166788284, 0.9261497926205429, 0.8680129728421762, 0.8137647753609764, 0.7661118752622963, 0.7198161600961848, 0.6806799293737783, 0.6465650990014783, 0.6215149611896057, 0.6055408403310509, 0.5897405537428495, 0.5729677728164664, 0.559595745790972, 0.5505845779251468, 0.5392379718024432, 0.5286539715907157, 0.5213795493986563, 0.5112658018609328, 0.504179925971016, 0.49809098381419403, 0.49417516361615327 ], "peak_layer": 1, "peak_normalized": 2.495529360167265, "population_cv": 0.5694475109215509, "rest_mean": 2.533740560769123e-06, "spike_contrast": 0.5337459133346284, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3523736697607091e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.862377747689607e-06, 5.270805104373721e-06, 4.903894932795083e-06, 4.609466031979537e-06, 4.09199901696411e-06, 3.725753003891441e-06, 3.479480710666394e-06, 3.250749841754441e-06, 2.9308416742424015e-06, 2.718335281315376e-06, 2.5433141672692727e-06, 2.3790946670487756e-06, 2.175666622861172e-06, 2.0390943973325193e-06, 1.911657136588474e-06, 1.7997132317759679e-06, 1.690957560640527e-06, 1.5990206065907842e-06, 1.5188796851361985e-06, 1.4600331041947356e-06, 1.4225074664864223e-06, 1.385390191899205e-06, 1.345988380307972e-06, 1.3145754564902745e-06, 1.2934068536196719e-06, 1.2667519513342995e-06, 1.2418885262377444e-06, 1.2247998029124574e-06, 1.2010410728180432e-06, 1.1843952734125196e-06, 1.1700914228640613e-06, 1.1608925660766545e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.614924046109081e-05, "first_to_last_ratio": 2.650607057187456, "imbalance_abs_log_ratio": 0.9747886919476757, "last_quartile_mean": 1.7410819282304146e-05, "mean": 2.826828892921185e-05, "normalized": [ 1.9567413134368976, 1.8597988174558218, 1.7664389243795566, 1.6893991915938202, 1.5636317866388034, 1.481959302241336, 1.406542023468853, 1.33584488000565, 1.2332974726907746, 1.1620116796950406, 1.1014492434217877, 1.0446845875825088, 0.983522176643611, 0.9354772941758706, 0.8921357523597812, 0.8494665759749792, 0.814434979275871, 0.7792464393968499, 0.7495682845985361, 0.7246339449332857, 0.7117101663246612, 0.6923010116299827, 0.676425813245603, 0.6619707608147611, 0.6525593178077636, 0.6397085734382539, 0.6282016590684582, 0.6192136216062852, 0.6075928767792258, 0.600035924319621, 0.59270322236514, 0.5872923826306096 ], "peak_layer": 1, "peak_normalized": 1.9567413134368976, "population_cv": 0.41380204518157304, "rest_mean": 2.9948716137032198e-05, "spike_contrast": 0.6408949859205556, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9193982006981968e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.5313728807959706e-05, 5.2573330322047696e-05, 4.9934205890167505e-05, 4.7756424464751035e-05, 4.420119512360543e-05, 4.189245373709127e-05, 3.976053631049581e-05, 3.776204903260805e-05, 3.486320929368958e-05, 3.284808190073818e-05, 3.113608545390889e-05, 2.9531445761676878e-05, 2.7802489057648927e-05, 2.6444342438480817e-05, 2.5219151211786084e-05, 2.4012966605369e-05, 2.3022683308226988e-05, 2.2027963495929725e-05, 2.1189012841205113e-05, 2.0484161723288707e-05, 2.0118828615522943e-05, 1.9570165022742003e-05, 1.91214003280038e-05, 1.8712780729401857e-05, 1.844673533923924e-05, 1.80834667844465e-05, 1.7758186004357412e-05, 1.7504109564470127e-05, 1.7175610992126167e-05, 1.696198887657374e-05, 1.6754705939092673e-05, 1.660175075812731e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.735129757842515e-05, "first_to_last_ratio": 2.0645564856398715, "imbalance_abs_log_ratio": 0.724915426415284, "last_quartile_mean": 1.8091681113219238e-05, "mean": 2.5016603217409283e-05, "normalized": [ 1.7802727363091704, 1.6959341879966399, 1.614090945613871, 1.5398204382964547, 1.4345125366505216, 1.3604640896020457, 1.2905915971592592, 1.2287960198423467, 1.1574098238374129, 1.0917198300160997, 1.0408054967224047, 0.9954414340603451, 0.9640031824094093, 0.9321987567197101, 0.8975491383896601, 0.8692109324382222, 0.8490852481619621, 0.8275443114054619, 0.8048684241781405, 0.7865739736521292, 0.7803940959907774, 0.7668844843835221, 0.7577356594722136, 0.7485970141410137, 0.7465552812163766, 0.7393132370819799, 0.7324712504458508, 0.7265002723105846, 0.7169511717671491, 0.7123002670381074, 0.7073969087223727, 0.7040072539687854 ], "peak_layer": 1, "peak_normalized": 1.7802727363091704, "population_cv": 0.3192203007098056, "rest_mean": 2.612829794762311e-05, "spike_contrast": 0.7276957615979828, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.901345167425461e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.453637666301802e-05, 4.2426512663951144e-05, 4.0379072743235156e-05, 3.852107693091966e-05, 3.588663093978539e-05, 3.4034190321108326e-05, 3.228621790185571e-05, 3.074030246352777e-05, 2.8954462322872132e-05, 2.7311121812090278e-05, 2.6037418138002977e-05, 2.4902563382056542e-05, 2.4116085114656016e-05, 2.3320446416619234e-05, 2.2453630663221702e-05, 2.1744705009041354e-05, 2.12412287510233e-05, 2.070234768325463e-05, 2.013507400988601e-05, 1.967740899999626e-05, 1.952280945261009e-05, 1.9184844859410077e-05, 1.8955972336698323e-05, 1.8727354472503066e-05, 1.86762772500515e-05, 1.849510590545833e-05, 1.8323942640563473e-05, 1.817456904973369e-05, 1.7935682990355417e-05, 1.781933315214701e-05, 1.769666778272949e-05, 1.761187013471499e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010835385364771355, "first_to_last_ratio": 4.77251940630934, "imbalance_abs_log_ratio": 1.5628743428869922, "last_quartile_mean": 2.2703701006321353e-05, "mean": 5.5646100065587234e-05, "normalized": [ 2.403350415322374, 2.2674490142770183, 2.122035773013255, 2.0228671307801673, 1.8424615989793292, 1.7404558535507528, 1.6337825778074309, 1.5451642251234987, 1.388568890335992, 1.2841278491651367, 1.2075395305627292, 1.1167532743401667, 1.0024609552195234, 0.9243915845434088, 0.8572737878531238, 0.7853215531962799, 0.7194510201884433, 0.662862527008837, 0.6162760059182385, 0.5774974679283221, 0.5437012181417744, 0.5145768135563884, 0.4900744695357144, 0.4675432113937477, 0.44713840772999236, 0.4307423296777627, 0.4166278785720478, 0.407689035656545, 0.39881798875942465, 0.3914264902634726, 0.3871640362552815, 0.38440708534382073 ], "peak_layer": 1, "peak_normalized": 2.403350415322374, "population_cv": 0.6252801969720955, "rest_mean": 6.087470218416355e-05, "spike_contrast": 0.4502962255543685, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.741164862527512e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013373707770369947, 0.0001261746947420761, 0.00011808301496785134, 0.00011256466677878052, 0.00010252580250380561, 9.684958058642223e-05, 9.091362881008536e-05, 8.598236308898777e-05, 7.726844341959804e-05, 7.14567067916505e-05, 6.719486555084586e-05, 6.214296445250511e-05, 5.5783042625989765e-05, 5.143878661328927e-05, 4.770394298247993e-05, 4.370008173282258e-05, 4.003464346169494e-05, 3.688571450766176e-05, 3.429335629334673e-05, 3.213548188796267e-05, 3.0254852390498854e-05, 2.8634192858589813e-05, 2.7270732971373945e-05, 2.601695632620249e-05, 2.48815085797105e-05, 2.396913077973295e-05, 2.31837166211335e-05, 2.2686304873786867e-05, 2.219266571046319e-05, 2.1781357645522803e-05, 2.1544168703258038e-05, 2.139075513696298e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bbef604712053bff665de420d5353433a19666047adbe681202c1aa4c9b40d0a", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.028617795556783676, 0.015273182652890682, 0.015790818259119987, 0.01658804342150688, 0.017823126167058945, 0.012884888797998428, 0.013985250145196915, 0.015267650596797466, 0.016450626775622368, 0.013461960479617119, 0.014477741904556751, 0.01562186423689127, 0.016906939446926117, 0.014646334573626518, 0.015772713348269463, 0.016963548958301544, 0.018231164664030075, 0.016293151304125786, 0.017603756859898567, 0.018859725445508957, 0.020210443064570427, 0.018590020015835762, 0.019812360405921936, 0.021170100197196007, 0.022594628855586052, 0.021153371781110764, 0.022732628509402275, 0.024445844814181328, 0.02632913924753666, 0.02505173534154892, 0.026562821120023727, 0.028511708602309227 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.69130083552227e-06, "first_to_last_ratio": 6.293146230018865, "imbalance_abs_log_ratio": 1.839461141207743, "last_quartile_mean": 4.2765585561710395e-07, "mean": 1.3660126239045667e-06, "normalized": [ 1.7482498804289475, 2.815312168363761, 2.2948186868978717, 1.7697692849656192, 1.588517521198396, 2.217468575156241, 1.8408222360962587, 1.4865407043733083, 1.34895693309848, 1.581801071425344, 1.3164943144629158, 1.2058085504791274, 0.9210663509923345, 1.0910843610686138, 0.9814732856417313, 0.7613599580616169, 0.7590642708352797, 0.7736743925325051, 0.5757114422524934, 0.5783657057699658, 0.4762344643636547, 0.48455133703992814, 0.46520203272174326, 0.4131025246748022, 0.35882542613057594, 0.3804709170270179, 0.35152785556076144, 0.33882867043560416, 0.26176283558069635, 0.288666522999825, 0.28234383302193, 0.24212388634264995 ], "peak_layer": 2, "peak_normalized": 2.815312168363761, "population_cv": 0.6846932882794533, "rest_mean": 1.5077786391278398e-06, "spike_contrast": 0.39825218776559335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.004761416988913e-07, "top_five_layers": [ 2, 3, 6, 7, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.3881314064055914e-06, 3.845751962217037e-06, 3.1347512958745938e-06, 2.4175271846615942e-06, 2.169934987250599e-06, 3.0290900667750975e-06, 2.514586412871722e-06, 2.0306333681219257e-06, 1.8426921997161116e-06, 2.1607602320727892e-06, 1.7983478528549313e-06, 1.6471497019665549e-06, 1.2581882629092433e-06, 1.4904350109645748e-06, 1.3407048982116976e-06, 1.0400273140476202e-06, 1.0368913763159071e-06, 1.056848986991099e-06, 7.864290978432109e-07, 7.900548553152476e-07, 6.505422902591818e-07, 6.619032433263783e-07, 6.354718493639666e-07, 5.643032636726275e-07, 4.901600618723023e-07, 5.197280756874534e-07, 4.801914883501013e-07, 4.6284424115583533e-07, 3.575713378722867e-07, 3.9432211451639887e-07, 3.856852401895594e-07, 3.3074428529289435e-07 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.969959031266626e-06, "first_to_last_ratio": 5.9310438749867265, "imbalance_abs_log_ratio": 1.7802002303990685, "last_quartile_mean": 1.1751656501246543e-06, "mean": 3.538651533574466e-06, "normalized": [ 1.643447719443132, 2.664460681731711, 2.26682328846465, 1.9117870285326903, 1.6953623890774276, 2.1430603290971333, 1.8420439675939804, 1.5903367627347242, 1.322590649598985, 1.514836257012002, 1.3201556675944437, 1.1519184327291594, 0.9433654624583057, 1.1090475597261495, 0.9445312929810626, 0.7800763891189801, 0.6931905948073118, 0.7356932905122949, 0.6107506859415541, 0.565158192377895, 0.4854051424814719, 0.5215479448254272, 0.4630492306077362, 0.4246074405439294, 0.3827682671544914, 0.3985910238724124, 0.3689537794505018, 0.34747432873448575, 0.29323390945282946, 0.31234102613386355, 0.2913380855124665, 0.2620531796967914 ], "peak_layer": 2, "peak_normalized": 2.664460681731711, "population_cv": 0.6669213577125401, "rest_mean": 3.8954815493154995e-06, "spike_contrast": 0.4137535830085245, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6117694485728862e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 5.815588792756898e-06, 9.428597877558786e-06, 8.021497706067748e-06, 6.765148100384977e-06, 5.99929671807331e-06, 7.5835437201021705e-06, 6.518351710838033e-06, 5.627647624351084e-06, 4.680187430494698e-06, 5.360477643989725e-06, 4.671570877690101e-06, 4.076237928529736e-06, 3.3382416404492687e-06, 3.9245328480319586e-06, 3.3423671084165107e-06, 2.7604185106611112e-06, 2.4529599613742903e-06, 2.6033621907117777e-06, 2.1612338514387375e-06, 1.9998979041702114e-06, 1.7176796518469928e-06, 1.845576434789109e-06, 1.6385698700105422e-06, 1.5025377706479048e-06, 1.354483515569882e-06, 1.4104747378951288e-06, 1.3055988574706134e-06, 1.2295905662540463e-06, 1.037652623381291e-06, 1.1052660511268186e-06, 1.0309439630873385e-06, 9.273148862121161e-07 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.006181029370054603, "first_to_last_ratio": 5.931043870000395, "imbalance_abs_log_ratio": 1.7802002295583512, "last_quartile_mean": 0.00104214865132235, "mean": 0.003138111591397319, "normalized": [ 1.6434476617442173, 2.664460684658614, 2.2668230869277246, 1.9117870850052245, 1.6953623005090255, 2.143060416265084, 1.8420439512820037, 1.5903368591312919, 1.3225906951513633, 1.514836380023738, 1.3201556314188725, 1.1519184729978353, 0.9433654289575268, 1.1090475511276263, 0.9445313214466298, 0.7800763764287361, 0.6931906017734022, 0.735693296303442, 0.6107506732925484, 0.5651581485023144, 0.4854051370092815, 0.5215479525604865, 0.463049256112769, 0.42460744955562396, 0.3827682394079058, 0.39859099715083124, 0.3689537785909722, 0.3474743574102401, 0.293233887776444, 0.3123410323265631, 0.2913381068209067, 0.2620531823307552 ], "peak_layer": 2, "peak_normalized": 2.664460684658614, "population_cv": 0.6669213544088491, "rest_mean": 0.0034545520191184348, "spike_contrast": 0.41375358477538404, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0014293332817032934, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.005157322157174349, 0.008361374959349632, 0.007113543804734945, 0.005999401211738586, 0.005320236086845398, 0.006725162733346224, 0.005780539475381374, 0.004990654531866312, 0.004150437191128731, 0.004753725603222847, 0.004142795689404011, 0.0036148487124592066, 0.002960385987535119, 0.003480314975604415, 0.002964044688269496, 0.002447966719046235, 0.0021753094624727964, 0.002308687660843134, 0.0019166037673130631, 0.0017735293367877603, 0.00152325548697263, 0.0016366756753996015, 0.0014531002379953861, 0.0013324655592441559, 0.0012011694489046931, 0.0012508230283856392, 0.00115781812928617, 0.0010904133087024093, 0.0009202006622217596, 0.0009801610140129924, 0.0009142514900304377, 0.000822352129034698 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7807550679881388e-06, "first_to_last_ratio": 5.412775845237589, "imbalance_abs_log_ratio": 1.68876205650804, "last_quartile_mean": 5.137391880793984e-07, "mean": 1.481595617747189e-06, "normalized": [ 1.4558359312069442, 2.4048173635519636, 2.145458829580914, 1.8401177624478837, 1.622434601507838, 2.1139236833600807, 1.8608213201971038, 1.571511096081306, 1.3844269423754563, 1.5890147466157112, 1.3681012923101372, 1.2125713912683727, 0.9918873677472284, 1.167668026852955, 0.9903155750421241, 0.8104020925085065, 0.7254100339117308, 0.7742380459295685, 0.633010046622503, 0.5955680935941017, 0.5095544541449822, 0.535113273728584, 0.48051365792731365, 0.4433064686062578, 0.3985779012901482, 0.41577980046677077, 0.38175667305719, 0.36725377000482734, 0.30148202599237856, 0.322715116130679, 0.30949829156073677, 0.2769143243777028 ], "peak_layer": 2, "peak_normalized": 2.4048173635519636, "population_cv": 0.6256905046703491, "rest_mean": 1.6260750191610929e-06, "spike_contrast": 0.4313496252306795, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.014068501121074e-07, "top_five_layers": [ 2, 3, 6, 7, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.1569601358351065e-06, 3.562966867320938e-06, 3.178702399964095e-06, 2.7263104129815474e-06, 2.4037919956754195e-06, 3.1319800655182917e-06, 2.7569847134145675e-06, 2.3283439531951444e-06, 2.051160890914616e-06, 2.3542772851214977e-06, 2.026972879320965e-06, 1.796540459508833e-06, 1.469575977353088e-06, 1.730011831568845e-06, 1.4672472161691985e-06, 1.2006881888737553e-06, 1.07476432731346e-06, 1.1471076959423954e-06, 9.378649110658444e-07, 8.823910775390686e-07, 7.549536462647666e-07, 7.928214813546219e-07, 7.119269298527797e-07, 6.568009212060133e-07, 5.905312718823552e-07, 6.160175303193682e-07, 5.656090138472791e-07, 5.441215762402862e-07, 4.4667444853985216e-07, 4.781333018399891e-07, 4.585513124766294e-07, 4.1027504948942806e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.0954280109417596e-06, "first_to_last_ratio": 5.491512307311366, "imbalance_abs_log_ratio": 1.7032036834003335, "last_quartile_mean": 7.457741659777639e-07, "mean": 2.1618307570747675e-06, "normalized": [ 1.4068765300276402, 2.3612070010793174, 2.1657643431477385, 1.8665928880193514, 1.6616002088094273, 2.1771057296508944, 1.869426341469116, 1.6468335409960086, 1.3542408725522084, 1.5667520899899137, 1.3594800268466822, 1.1819651148069434, 1.009334596910006, 1.1450136844580612, 0.9957592626981273, 0.817213831842472, 0.712537815574818, 0.75485624218804, 0.6404857973819618, 0.5842511071078107, 0.5073510858223989, 0.539798913350487, 0.47838870053156934, 0.4373768546103689, 0.39708015593236357, 0.4165380876399892, 0.3871856292058109, 0.3596150701163213, 0.30484043926272086, 0.32078253274025254, 0.30118994888789563, 0.2725555563432833 ], "peak_layer": 2, "peak_normalized": 2.3612070010793174, "population_cv": 0.6307038510118401, "rest_mean": 2.3732101079042842e-06, "spike_contrast": 0.4299586704088529, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0203822625953763e-06, "top_five_layers": [ 2, 6, 3, 7, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.0414289540203754e-06, 5.104529918753542e-06, 4.682015969592612e-06, 4.035257916257251e-06, 3.592098437366076e-06, 4.706534127763007e-06, 4.041383363073692e-06, 3.5601754007075215e-06, 2.9276395707711345e-06, 3.387052856851369e-06, 2.938965735665988e-06, 2.555208538979059e-06, 2.1820105757797137e-06, 2.4753258003329393e-06, 2.152663000742905e-06, 1.766677996783983e-06, 1.5403861652885098e-06, 1.6318714415319846e-06, 1.3846218962498824e-06, 1.2630520132006495e-06, 1.0968071819661418e-06, 1.1669538935166202e-06, 1.0341954066461767e-06, 9.455347367293143e-07, 8.584200941186282e-07, 9.004848493532336e-07, 8.370298019144684e-07, 7.774269192850625e-07, 6.590134375983325e-07, 6.934775456102216e-07, 6.5112169522763e-07, 5.892189847145346e-07 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.600380369699451e-06, "first_to_last_ratio": 5.360934028391998, "imbalance_abs_log_ratio": 1.6791382189297277, "last_quartile_mean": 4.850610650919407e-07, "mean": 1.407810660936093e-06, "normalized": [ 1.2424006186409213, 2.116197835877747, 2.1289350583166167, 1.8206110660116281, 1.672772135624017, 2.180903391012272, 1.9110341366235388, 1.7040213242162747, 1.3715763299737698, 1.620181348382822, 1.418229895744138, 1.2018589460048466, 1.0672724508221327, 1.18033346969631, 1.0372483057503512, 0.8599070649138534, 0.7226324996147198, 0.7616597051326, 0.6571229137740889, 0.5957180716477494, 0.5220566569858939, 0.5404710315048392, 0.4712153724045113, 0.439240928976565, 0.4052289724922489, 0.41792517172028115, 0.3921276690571982, 0.35367305227247126, 0.30647629235309304, 0.31562180347619373, 0.2957780769261062, 0.2695684040502012 ], "peak_layer": 6, "peak_normalized": 2.180903391012272, "population_cv": 0.6156495501618916, "rest_mean": 1.5445136143541646e-06, "spike_contrast": 0.43354406607707613, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.696147124785057e-07, "top_five_layers": [ 6, 3, 2, 7, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.7490648360762862e-06, 2.9792058739985805e-06, 2.9971374715387356e-06, 2.563075668149395e-06, 2.354946445848327e-06, 3.0702990443387534e-06, 2.69037423095142e-06, 2.39893938669411e-06, 1.9309197796246735e-06, 2.280908574903151e-06, 1.996599166886881e-06, 1.6919898371270392e-06, 1.5025175343907904e-06, 1.6616860420981538e-06, 1.4602492228732444e-06, 1.2105863333999878e-06, 1.0173297368964995e-06, 1.0722726528911153e-06, 9.251046435565513e-07, 8.386582521779928e-07, 7.349569273173984e-07, 7.608808800796396e-07, 6.633820248680422e-07, 6.183680625326815e-07, 5.704856675947667e-07, 5.883595122213592e-07, 5.520415129467438e-07, 4.979046934749931e-07, 4.314605916988512e-07, 4.4433573975766194e-07, 4.163995299677481e-07, 3.7950127307340153e-07 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 1.2294863836359582e-05, "first_to_last_ratio": 5.599817429618977, "imbalance_abs_log_ratio": 1.7227339953558973, "last_quartile_mean": 2.195582979425126e-06, "mean": 6.332394622887705e-06, "normalized": [ 1.5057250244887312, 2.556447232676371, 2.2172679267612243, 1.8985687859696891, 1.6555718867405833, 2.216307645234537, 1.8634142379769214, 1.619353790332452, 1.3446938842284024, 1.5146763535149452, 1.3286277681864072, 1.142126618231696, 0.9485689933012624, 1.1114498891377516, 0.9384618363381629, 0.7856483849547331, 0.6834110772125562, 0.720452579841522, 0.6239316473465666, 0.58489030026986, 0.4861726124726784, 0.5510752056898781, 0.49322129575339285, 0.4361559268955639, 0.38962144676483523, 0.4232163990310226, 0.3815927693281277, 0.36416773858904095, 0.3092119801312069, 0.32838130926563275, 0.29808727088779047, 0.27950018244645464 ], "peak_layer": 2, "peak_normalized": 2.556447232676371, "population_cv": 0.6506449880493771, "rest_mean": 6.95244242015852e-06, "spike_contrast": 0.4292213206934063, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.984136517625302e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 9.5348450486199e-06, 1.6188432709896006e-05, 1.4040615496924147e-05, 1.2022486771456897e-05, 1.0483734513400123e-05, 1.4034534615348093e-05, 1.1799874300777446e-05, 1.0254387234454043e-05, 8.515132321917918e-06, 9.591528396413196e-06, 8.413395335082896e-06, 7.23239645594731e-06, 6.006713192618918e-06, 7.0381393015850335e-06, 5.942710686213104e-06, 4.975035608367762e-06, 4.3276286305626854e-06, 4.562190042634029e-06, 3.950981408706866e-06, 3.7037561924080364e-06, 3.0786368370172568e-06, 3.48962566931732e-06, 3.1232718811224913e-06, 2.7619114462140715e-06, 2.467236754455371e-06, 2.679973249541945e-06, 2.416396000626264e-06, 2.306053829670418e-06, 1.9580522803153144e-06, 2.0794400370505173e-06, 1.887606231321115e-06, 1.769905452420062e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4e4a33d69f29ecf2b2276b1faef8a4abb641becd86149078c17228416d2e6c90", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014952295459806919, 0.014778315089643002, 0.01543855108320713, 0.016689632087945938, 0.01178102195262909, 0.012982882559299469, 0.014135614968836308, 0.015412751585245132, 0.012501097284257412, 0.013585822656750679, 0.014657454565167427, 0.01575874350965023, 0.013761760666966438, 0.014892088249325752, 0.016054842621088028, 0.017220331355929375, 0.015453077852725983, 0.016655873507261276, 0.017766626551747322, 0.018939858302474022, 0.017597980797290802, 0.018625281751155853, 0.01988193951547146, 0.0208931565284729, 0.019704710692167282, 0.021027294918894768, 0.02250491827726364, 0.023921677842736244, 0.022959763184189796, 0.02448389306664467, 0.025734128430485725, 0.027005694806575775 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.67974108839553e-06, "first_to_last_ratio": 3.8051990936026456, "imbalance_abs_log_ratio": 1.3363683141504743, "last_quartile_mean": 2.0182231992293964e-06, "mean": 4.804195157248614e-06, "normalized": [ 1.5877510011530913, 1.6138570201166533, 1.5124622299167279, 1.3859304173236653, 1.9450590445309985, 1.723923747485618, 1.5835986184675985, 1.4358094613589012, 1.674997608449353, 1.5338764199258852, 1.3688100265161884, 1.2248142207369597, 1.332209841900902, 1.1411212950118081, 0.9749533862079496, 0.9261620377466854, 0.9577260337784566, 0.8878246260085111, 0.7898909908438367, 0.6795963310980445, 0.6887310902589849, 0.63469625591136, 0.534405205528709, 0.5010250390176237, 0.5713704795033905, 0.4976710822451457, 0.45953689758880223, 0.3909391200832928, 0.4202843960445734, 0.3737497958233971, 0.34227896525752816, 0.30493731415935826 ], "peak_layer": 5, "peak_normalized": 1.9450590445309985, "population_cv": 0.4919814241796063, "rest_mean": 5.172476204717424e-06, "spike_contrast": 0.5443190822896888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8154775009170407e-06, "top_five_layers": [ 5, 6, 9, 2, 1 ], "top_five_spike_overlap": 0 }, "values": [ 7.62786567065632e-06, 7.753284080536105e-06, 7.266163720487384e-06, 6.658280199189903e-06, 9.344443242298439e-06, 8.282066119136289e-06, 7.607916813867632e-06, 6.897908860992175e-06, 8.047015398915391e-06, 7.36904166842578e-06, 6.576030500582419e-06, 5.884246547793737e-06, 6.4001960708992556e-06, 5.4821693993289955e-06, 4.683866336563369e-06, 4.449463176570134e-06, 4.601102773449384e-06, 4.265282768756151e-06, 3.7947904729662696e-06, 3.264913402745151e-06, 3.3087985684687737e-06, 3.049204678973183e-06, 2.567386900409474e-06, 2.407022066108766e-06, 2.744975290625007e-06, 2.390909003224806e-06, 2.207704937973176e-06, 1.8781478274831898e-06, 2.019128260144498e-06, 1.7955669591174228e-06, 1.6443749473182834e-06, 1.4649783679487882e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.071156265832542e-05, "first_to_last_ratio": 3.7446231103146244, "imbalance_abs_log_ratio": 1.3203209738045956, "last_quartile_mean": 8.20150967228983e-06, "mean": 1.9372753484958594e-05, "normalized": [ 1.5605669500883477, 1.6008805234793573, 1.4979441121615527, 1.3667092062703194, 1.935846243370447, 1.7068521590622452, 1.5728591107226166, 1.4407154966572362, 1.6735980219259619, 1.5302773348762577, 1.3683676597193748, 1.2238142355733308, 1.331126586595219, 1.1475793660881108, 0.9834211123659795, 0.934717997327087, 0.9606142085118456, 0.8999286898408035, 0.7947106491954754, 0.6878868767005736, 0.6987226911749845, 0.6477891191535685, 0.5408853487908869, 0.5073638009416495, 0.5749202632388171, 0.5019822542376574, 0.4637585312481849, 0.3931847315331312, 0.422636716488808, 0.3778631680748615, 0.346655525688182, 0.3058213088971263 ], "peak_layer": 5, "peak_normalized": 1.935846243370447, "population_cv": 0.485582780500018, "rest_mean": 2.0829526268547244e-05, "spike_contrast": 0.552397606418649, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1506180453579872e-05, "top_five_layers": [ 5, 6, 9, 2, 7 ], "top_five_spike_overlap": 0 }, "values": [ 3.023247882083524e-05, 3.101346374023706e-05, 2.9019302019150928e-05, 2.6476920538698323e-05, 3.750267205759883e-05, 3.306642611278221e-05, 3.0470611818600446e-05, 2.7910626158700325e-05, 3.242220191168599e-05, 2.9645685572177172e-05, 2.6509049348533154e-05, 2.3708651497145183e-05, 2.5787587219383568e-05, 2.223177216365002e-05, 1.9051574781769887e-05, 1.8108061340171844e-05, 1.8609742255648598e-05, 1.7434096662327647e-05, 1.5395733498735353e-05, 1.332626288785832e-05, 1.3536182450479828e-05, 1.2549458915600553e-05, 1.0478438525751699e-05, 9.829033842834178e-06, 1.1137788533233106e-05, 9.724778465169948e-06, 8.984279702417552e-06, 7.617070878040977e-06, 8.187636922230013e-06, 7.3202500061597675e-06, 6.71567204335588e-06, 5.9246008277114015e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027235321002081037, "first_to_last_ratio": 3.744623083996792, "imbalance_abs_log_ratio": 1.3203209667764297, "last_quartile_mean": 0.00727318087592721, "mean": 0.017179951537400484, "normalized": [ 1.5605669970711773, 1.600880476363625, 1.497944105038087, 1.366709216880459, 1.935846276047663, 1.7068521655411573, 1.5728590802784164, 1.4407155266358058, 1.6735980011744565, 1.530277352493985, 1.3683676045738475, 1.2238142239324827, 1.331126522669949, 1.1475793472944473, 0.983421191686974, 0.9347179793604132, 0.9606142409543752, 0.8999287004964132, 0.7947106908462469, 0.6878868770559932, 0.6987226668860359, 0.6477890987517776, 0.540885325434597, 0.507363798093792, 0.5749202742530459, 0.5019822779872913, 0.46375853415484747, 0.39318471070846445, 0.4226367318074797, 0.3778631643128976, 0.34665553015673556, 0.3058213110570609 ], "peak_layer": 5, "peak_normalized": 1.935846276047663, "population_cv": 0.4855827790155644, "rest_mean": 0.018471832249175618, "spike_contrast": 0.5523975941406755, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.010203795693814754, "top_five_layers": [ 5, 6, 9, 2, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.02681046538054943, 0.02750304900109768, 0.025734607130289078, 0.023479998111724854, 0.03325774520635605, 0.02932363748550415, 0.02702164277434349, 0.02475142292678356, 0.02875233255326748, 0.02629009075462818, 0.02350848913192749, 0.021025069057941437, 0.02286868914961815, 0.019715357571840286, 0.016895128414034843, 0.016058409586548805, 0.016503306105732918, 0.015460731461644173, 0.01365309115499258, 0.011817863211035728, 0.012004021555185318, 0.011128985323011875, 0.009292383678257465, 0.00871648546308279, 0.009877102449536324, 0.008624031208455563, 0.007967349141836166, 0.006754894275218248, 0.007260878570377827, 0.006491670850664377, 0.005955525208264589, 0.005253995303064585 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.0114431578122094e-06, "first_to_last_ratio": 3.7338115900101996, "imbalance_abs_log_ratio": 1.3174295858952805, "last_quartile_mean": 5.387104060616821e-07, "mean": 1.2825281023864932e-06, "normalized": [ 1.5532146686344612, 1.5705007170330607, 1.4609122866463597, 1.3427289650819811, 1.9488030425764962, 1.6741560212020503, 1.555147967348224, 1.4412752548233487, 1.6499717927986355, 1.4946710860708503, 1.3520299845289774, 1.2293314664225108, 1.3640908964772576, 1.20475774170785, 0.9983558990009345, 0.9578823214532782, 0.9984385140583039, 0.9052889731611042, 0.8076018091021662, 0.7077176784237299, 0.6965311830349138, 0.6602340537013012, 0.5446056382431776, 0.5214488681337209, 0.5862688460241238, 0.5169358876607686, 0.44543689900304595, 0.3852469543573297, 0.4223793644694745, 0.37423357828937454, 0.3347806778641219, 0.2950209626670668 ], "peak_layer": 5, "peak_normalized": 1.9488030425764962, "population_cv": 0.4798785088756173, "rest_mean": 1.3770984666010648e-06, "spike_contrast": 0.5604887045825205, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.718481356278062e-07, "top_five_layers": [ 5, 6, 9, 2, 7 ], "top_five_spike_overlap": 0 }, "values": [ 1.9920414615626214e-06, 2.0142113044130383e-06, 1.8736610627456685e-06, 1.722087631605973e-06, 2.499394668120658e-06, 2.1471521449711872e-06, 1.99452097149333e-06, 1.8484760175851989e-06, 2.116135192409274e-06, 1.9169576717104064e-06, 1.734016450427589e-06, 1.5766521528348676e-06, 1.7494849089416675e-06, 1.5451356603080058e-06, 1.28041949665203e-06, 1.2285109960430418e-06, 1.2805254527847865e-06, 1.161058548859728e-06, 1.0357720157117e-06, 9.076678111341607e-07, 8.933208164307871e-07, 8.467687280244718e-07, 6.984720357650076e-07, 6.687728273391258e-07, 7.519062705796387e-07, 6.629848030570429e-07, 5.712853408113006e-07, 4.94090045322082e-07, 5.417134048002481e-07, 4.799650810127787e-07, 4.293656274967361e-07, 3.783726754136296e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.3353832602879265e-05, "first_to_last_ratio": 3.2227511873122405, "imbalance_abs_log_ratio": 1.1702354007568778, "last_quartile_mean": 7.246551547268609e-06, "mean": 1.5164828951697018e-05, "normalized": [ 1.5418197968348877, 1.5709075226993274, 1.4767167540770285, 1.3358674513994462, 1.888773418938626, 1.6450250100697903, 1.500898730733707, 1.3599892142097643, 1.6017342532341532, 1.4490627654746335, 1.298243393256253, 1.1640713503380204, 1.2947764191223834, 1.1168088786006158, 0.9842365208656229, 0.9219899773989011, 0.9795395409797548, 0.9070325940775823, 0.8195392964359715, 0.7255652736709899, 0.7453740698008642, 0.6952593569228117, 0.5937884104162011, 0.5601599368070976, 0.6362245384563836, 0.5594724554118488, 0.5161812187846071, 0.44522117998918276, 0.483203222643258, 0.4278241163331062, 0.3991997791976546, 0.3554935528195254 ], "peak_layer": 5, "peak_normalized": 1.888773418938626, "population_cv": 0.44033722604682474, "rest_mean": 1.615851486483869e-05, "spike_contrast": 0.6064248541835164, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.798925020731986e-06, "top_five_layers": [ 5, 6, 9, 2, 1 ], "top_five_spike_overlap": 0 }, "values": [ 2.338143349334132e-05, 2.38225438806694e-05, 2.2394156985683367e-05, 2.025820140261203e-05, 2.8642925826716237e-05, 2.4946522898972034e-05, 2.2760872525395826e-05, 2.062400380964391e-05, 2.429002597637009e-05, 2.197478897869587e-05, 1.9687638996401802e-05, 1.7652942915447056e-05, 1.9635062926681712e-05, 1.69362156157149e-05, 1.4925778486940544e-05, 1.3981820302433334e-05, 1.4854549590381794e-05, 1.375499414280057e-05, 1.2428173249645624e-05, 1.1003073268511798e-05, 1.1303470273560379e-05, 1.0543489224801306e-05, 9.004699677461758e-06, 8.494729627273045e-06, 9.64823630056344e-06, 8.484304089506622e-06, 7.827799890947063e-06, 6.7517030402086675e-06, 7.327694220293779e-06, 6.487879545602482e-06, 6.0537963690876495e-06, 5.390998921939172e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7634835330682108e-05, "first_to_last_ratio": 2.77858968586119, "imbalance_abs_log_ratio": 1.0219434917345125, "last_quartile_mean": 6.346685665903351e-06, "mean": 1.1861588987471805e-05, "normalized": [ 1.541845373328513, 1.5495572539383986, 1.4327425587355114, 1.2888012556294268, 1.830222537296193, 1.5687089838516481, 1.401198821657203, 1.280665512186601, 1.5188285787114204, 1.4028344660045762, 1.232338325101161, 1.090834003257619, 1.2372257058599228, 1.094748447061392, 1.0061819707458282, 0.9121581837805397, 1.0091865816307033, 0.8969224311834834, 0.8201245892341719, 0.7702972436236826, 0.7896771601850562, 0.7574046980046742, 0.6630426948751132, 0.62395653030935, 0.7012582077643145, 0.6230517196436284, 0.5687327928300282, 0.4928440937190253, 0.5522704219858736, 0.47317300148686847, 0.45736835411973276, 0.41179750225833966 ], "peak_layer": 5, "peak_normalized": 1.830222537296193, "population_cv": 0.3925976431843591, "rest_mean": 1.2505040962827883e-05, "spike_contrast": 0.6706853936328379, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.38694832054898e-06, "top_five_layers": [ 5, 6, 2, 1, 9 ], "top_five_spike_overlap": 0 }, "values": [ 1.8288736100657843e-05, 1.838021125877276e-05, 1.699460335657932e-05, 1.5287230780813843e-05, 2.170934749301523e-05, 1.8607381207402796e-05, 1.662044451222755e-05, 1.5190727935987525e-05, 1.801572034310084e-05, 1.663984585320577e-05, 1.4617490705859382e-05, 1.2939024600200355e-05, 1.467546280764509e-05, 1.2985456123715267e-05, 1.1934916983591393e-05, 1.0819645467563532e-05, 1.1970556442975067e-05, 1.0638925232342444e-05, 9.72798079601489e-06, 9.13694930204656e-06, 9.366825906909071e-06, 8.984023224911653e-06, 7.864739927754272e-06, 7.401115908578504e-06, 8.318036634591408e-06, 7.3903834163502324e-06, 6.746074632246746e-06, 5.845914074598113e-06, 6.550804755534045e-06, 5.612583663605619e-06, 5.425115432444727e-06, 4.884572717855917e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.088671580073424e-05, "first_to_last_ratio": 4.047770176202506, "imbalance_abs_log_ratio": 1.3981661557198917, "last_quartile_mean": 1.5042038739920827e-05, "mean": 3.823578526862548e-05, "normalized": [ 1.5715861755410512, 1.6089836583440513, 1.4748513229105977, 1.3543048727993656, 1.9303102550135327, 1.7240508454541201, 1.6068360247318012, 1.468287586760736, 1.6942393465207237, 1.561867876551647, 1.3928243162419038, 1.2823569584329257, 1.3510573524000515, 1.153964680955942, 0.9791255926207968, 0.9554361624959514, 0.9535104091594309, 0.9404174748271559, 0.8183623910744282, 0.6821935048854633, 0.6986677823476758, 0.637767260268953, 0.5187138398690193, 0.4930674009743013, 0.5397023628810284, 0.4874058383983964, 0.45262415468651734, 0.3647526912052027, 0.37717677353562973, 0.34769267169862494, 0.31586324682274186, 0.26199916959023306 ], "peak_layer": 5, "peak_normalized": 1.9303102550135327, "population_cv": 0.5019072738596447, "rest_mean": 4.1226788568935636e-05, "spike_contrast": 0.535680032656999, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.2084367446950637e-05, "top_five_layers": [ 5, 6, 9, 2, 7 ], "top_five_spike_overlap": 0 }, "values": [ 6.0090831539127976e-05, 6.15207536611706e-05, 5.639209848595783e-05, 5.1782910304609686e-05, 7.380692841252312e-05, 6.592043791897595e-05, 6.143863720353693e-05, 5.61411288799718e-05, 6.478057184722275e-05, 5.971924474579282e-05, 5.325573147274554e-05, 4.903192530036904e-05, 5.165873881196603e-05, 4.4122745748609304e-05, 3.743763591046445e-05, 3.6531851947074756e-05, 3.645821925601922e-05, 3.5957600630354136e-05, 3.1290728657040745e-05, 2.608420436445158e-05, 2.6714111299952492e-05, 2.4385532015003264e-05, 1.9833430997096002e-05, 1.885281926661264e-05, 2.0635943656088784e-05, 1.8636344975675456e-05, 1.73064399859868e-05, 1.3946605577075388e-05, 1.4421650121221319e-05, 1.3294302334543318e-05, 1.2077279279765207e-05, 1.0017743989010341e-05 ] } } } } } }, "step": 100 }, { "modes": { "learned": { "bits_per_byte": 3.0745070201808953, "forward": { "activation_sha256": { "attention_branch_output": "fe2bfa0ef8e4259f3ed7359d56fb5bbe56e2b0eafc30ccf1c1e3233cb3ec1708", "mlp_branch_output": "ace6376bf838ff2894b72bcf39de9a912ced4cfce16da0bea85a55c0a2d375f6", "post_attention_state": "7b7c9c05d1cbde4434ba686237dd1c945f87f81120c1f3f234a3adcdf1d68140", "post_mlp_state": "4c56d08d88ae21cddacd7e29fc552875e42d5e83a4e1d9907d475a90abe4ea95", "pre_attention_input": "787f9e4002b423fdbb7f8faf6fa82a8ff8c9854d40d44697d8afa4d95a3e169c", "pre_mlp_input": "ad0dc0ee191d8145bd7a67f8b576436d75a4c4558d60d541ecf9bca80ffb9f93" }, "logits_sha256": "7b3859e28e8af95527b68a3a79f7e5a03998e27718248cca6c3d4328f46ef140", "loss_nats": 2.1310858726501465, "mixer_summary_sha256": "9d8f4f3394602c9d71988fb3e74693f57cf5a16485049d9aa7c2782046516cad" }, "loss_nats": 2.1310858726501465, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6506600379943848, "group": 1, "latest_source_mass": 0.6252298355102539, "layer": 1, "max_source_mass": 0.6252298355102539, "mixer_index": 1, "normalized_entropy": 0.9387040101191234, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3747701644897461, "median": 0.35756751894950867, "p05": 0.2806163728237152, "p95": 0.5326682329177856 }, { "label": "current_group_partial", "mean": 0.6252298355102539, "median": 0.642432451248169, "p05": 0.46733176708221436, "p95": 0.7193835973739624 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6806463003158569, "group": 1, "latest_source_mass": 0.5466031432151794, "layer": 2, "max_source_mass": 0.5466031432151794, "mixer_index": 2, "normalized_entropy": 0.9819650420651069, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.45339685678482056, "median": 0.43950676918029785, "p05": 0.37430456280708313, "p95": 0.5817784070968628 }, { "label": "current_group_partial", "mean": 0.5466031432151794, "median": 0.5604932308197021, "p05": 0.418221652507782, "p95": 0.6256954669952393 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6869370937347412, "group": 1, "latest_source_mass": 0.5331428050994873, "layer": 2, "max_source_mass": 0.5331428050994873, "mixer_index": 3, "normalized_entropy": 0.9910407385337882, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4668571949005127, "median": 0.45914435386657715, "p05": 0.40756916999816895, "p95": 0.5482956171035767 }, { "label": "current_group_partial", "mean": 0.5331428050994873, "median": 0.5408556461334229, "p05": 0.45170438289642334, "p95": 0.5924307703971863 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6791321635246277, "group": 1, "latest_source_mass": 0.5272461175918579, "layer": 3, "max_source_mass": 0.5272461175918579, "mixer_index": 4, "normalized_entropy": 0.979780604425173, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4727538824081421, "median": 0.45147764682769775, "p05": 0.37644311785697937, "p95": 0.6218997836112976 }, { "label": "current_group_partial", "mean": 0.5272461175918579, "median": 0.5485223531723022, "p05": 0.3781002163887024, "p95": 0.6235568523406982 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6804035902023315, "group": 1, "latest_source_mass": 0.5512661933898926, "layer": 3, "max_source_mass": 0.5512661933898926, "mixer_index": 5, "normalized_entropy": 0.9816148853879503, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4487338662147522, "median": 0.4356163740158081, "p05": 0.36789458990097046, "p95": 0.5608916282653809 }, { "label": "current_group_partial", "mean": 0.5512661933898926, "median": 0.5643836259841919, "p05": 0.43910837173461914, "p95": 0.6321054100990295 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.682766318321228, "group": 1, "latest_source_mass": 0.5353820323944092, "layer": 4, "max_source_mass": 0.5353820323944092, "mixer_index": 6, "normalized_entropy": 0.9850235815280511, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.46461793780326843, "median": 0.44794324040412903, "p05": 0.38936740159988403, "p95": 0.5920549631118774 }, { "label": "current_group_partial", "mean": 0.5353820323944092, "median": 0.5520567893981934, "p05": 0.40794503688812256, "p95": 0.610632598400116 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6873732805252075, "group": 1, "latest_source_mass": 0.5116249322891235, "layer": 4, "max_source_mass": 0.5116249322891235, "mixer_index": 7, "normalized_entropy": 0.9916700230532952, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4883750379085541, "median": 0.48108816146850586, "p05": 0.41739019751548767, "p95": 0.5840339660644531 }, { "label": "current_group_partial", "mean": 0.5116249322891235, "median": 0.5189118981361389, "p05": 0.41596606373786926, "p95": 0.5826097726821899 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6835889220237732, "group": 2, "latest_source_mass": 0.5333441495895386, "layer": 5, "max_source_mass": 0.5333441495895386, "mixer_index": 8, "normalized_entropy": 0.9862103478103299, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.46665582060813904, "median": 0.45750027894973755, "p05": 0.3836124539375305, "p95": 0.5830957889556885 }, { "label": "completed_group_1", "mean": 0.5333441495895386, "median": 0.5424997806549072, "p05": 0.4169042110443115, "p95": 0.6163875460624695 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0856965780258179, "group": 2, "latest_source_mass": 0.37292569875717163, "layer": 5, "max_source_mass": 0.37292569875717163, "mixer_index": 9, "normalized_entropy": 0.9882436135336242, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27463093400001526, "median": 0.2607242465019226, "p05": 0.2352934032678604, "p95": 0.3491206765174866 }, { "label": "completed_group_1", "mean": 0.3524433374404907, "median": 0.3531361222267151, "p05": 0.3015327453613281, "p95": 0.39581894874572754 }, { "label": "current_group_partial", "mean": 0.37292569875717163, "median": 0.37286806106567383, "p05": 0.33192169666290283, "p95": 0.41239237785339355 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.077350378036499, "group": 2, "latest_source_mass": 0.3963770568370819, "layer": 6, "max_source_mass": 0.3963770568370819, "mixer_index": 10, "normalized_entropy": 0.9806465749100737, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2904614210128784, "median": 0.2787848711013794, "p05": 0.2332231104373932, "p95": 0.39111974835395813 }, { "label": "completed_group_1", "mean": 0.3131615221500397, "median": 0.3065500855445862, "p05": 0.2717450261116028, "p95": 0.37716877460479736 }, { "label": "current_group_partial", "mean": 0.3963770568370819, "median": 0.41992655396461487, "p05": 0.24805063009262085, "p95": 0.4772615134716034 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0850789546966553, "group": 2, "latest_source_mass": 0.3814815580844879, "layer": 6, "max_source_mass": 0.3814815580844879, "mixer_index": 11, "normalized_entropy": 0.9876814285521406, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.27783387899398804, "median": 0.2646753787994385, "p05": 0.2372770756483078, "p95": 0.3592980206012726 }, { "label": "completed_group_1", "mean": 0.34068456292152405, "median": 0.3382711410522461, "p05": 0.30354392528533936, "p95": 0.38547736406326294 }, { "label": "current_group_partial", "mean": 0.3814815580844879, "median": 0.38342589139938354, "p05": 0.31723207235336304, "p95": 0.4350103437900543 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.062077522277832, "group": 2, "latest_source_mass": 0.42493611574172974, "layer": 7, "max_source_mass": 0.42493611574172974, "mixer_index": 12, "normalized_entropy": 0.9667446224959214, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2839078903198242, "median": 0.2714773416519165, "p05": 0.22162550687789917, "p95": 0.3881691098213196 }, { "label": "completed_group_1", "mean": 0.29115602374076843, "median": 0.27455592155456543, "p05": 0.2443995624780655, "p95": 0.3912811577320099 }, { "label": "current_group_partial", "mean": 0.42493611574172974, "median": 0.4549126625061035, "p05": 0.2369934469461441, "p95": 0.5239657163619995 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0936429500579834, "group": 2, "latest_source_mass": 0.36200207471847534, "layer": 7, "max_source_mass": 0.36200207471847534, "mixer_index": 13, "normalized_entropy": 0.9954767130666716, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.30465999245643616, "median": 0.29826998710632324, "p05": 0.2748320400714874, "p95": 0.3600652515888214 }, { "label": "completed_group_1", "mean": 0.3333379328250885, "median": 0.33137744665145874, "p05": 0.30286699533462524, "p95": 0.36546915769577026 }, { "label": "current_group_partial", "mean": 0.36200207471847534, "median": 0.3605042099952698, "p05": 0.32377851009368896, "p95": 0.403035044670105 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0627415180206299, "group": 2, "latest_source_mass": 0.4155862033367157, "layer": 8, "max_source_mass": 0.4155862033367157, "mixer_index": 14, "normalized_entropy": 0.9673490174673293, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2850727438926697, "median": 0.2791768014431, "p05": 0.22466334700584412, "p95": 0.36319249868392944 }, { "label": "completed_group_1", "mean": 0.2993410527706146, "median": 0.27798235416412354, "p05": 0.22940492630004883, "p95": 0.4312986731529236 }, { "label": "current_group_partial", "mean": 0.4155862033367157, "median": 0.4446514844894409, "p05": 0.22757133841514587, "p95": 0.5349330306053162 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.087097406387329, "group": 2, "latest_source_mass": 0.3793410062789917, "layer": 8, "max_source_mass": 0.3793410062789917, "mixer_index": 15, "normalized_entropy": 0.9895187024580431, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28865373134613037, "median": 0.2799619734287262, "p05": 0.24506454169750214, "p95": 0.37004515528678894 }, { "label": "completed_group_1", "mean": 0.3320052921772003, "median": 0.33097362518310547, "p05": 0.2870577275753021, "p95": 0.3873226046562195 }, { "label": "current_group_partial", "mean": 0.3793410062789917, "median": 0.3775429427623749, "p05": 0.3292272984981537, "p95": 0.4380772113800049 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0500720739364624, "group": 3, "latest_source_mass": 0.4391520023345947, "layer": 9, "max_source_mass": 0.4391520023345947, "mixer_index": 16, "normalized_entropy": 0.9558167924823646, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27458876371383667, "median": 0.2675239145755768, "p05": 0.2081141322851181, "p95": 0.3727893829345703 }, { "label": "completed_group_1", "mean": 0.286259263753891, "median": 0.2620168924331665, "p05": 0.2130153328180313, "p95": 0.42261889576911926 }, { "label": "completed_group_2", "mean": 0.4391520023345947, "median": 0.4622455835342407, "p05": 0.24257506430149078, "p95": 0.5693691968917847 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3743348121643066, "group": 3, "latest_source_mass": 0.30083364248275757, "layer": 9, "max_source_mass": 0.30083364248275757, "mixer_index": 17, "normalized_entropy": 0.9913730090152552, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21668720245361328, "median": 0.21049442887306213, "p05": 0.1911841183900833, "p95": 0.26713764667510986 }, { "label": "completed_group_1", "mean": 0.23293715715408325, "median": 0.2271968424320221, "p05": 0.20331373810768127, "p95": 0.27224695682525635 }, { "label": "completed_group_2", "mean": 0.2495419979095459, "median": 0.2486393302679062, "p05": 0.22275272011756897, "p95": 0.2802416682243347 }, { "label": "current_group_partial", "mean": 0.30083364248275757, "median": 0.3044033646583557, "p05": 0.24597351253032684, "p95": 0.34015560150146484 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3609715700149536, "group": 3, "latest_source_mass": 0.3002921938896179, "layer": 10, "max_source_mass": 0.3002921938896179, "mixer_index": 18, "normalized_entropy": 0.9817334674257201, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.23144879937171936, "median": 0.22791123390197754, "p05": 0.20063085854053497, "p95": 0.2748033404350281 }, { "label": "completed_group_1", "mean": 0.21839870512485504, "median": 0.19323360919952393, "p05": 0.15541185438632965, "p95": 0.3410910665988922 }, { "label": "completed_group_2", "mean": 0.24986030161380768, "median": 0.24544566869735718, "p05": 0.21982765197753906, "p95": 0.28961730003356934 }, { "label": "current_group_partial", "mean": 0.3002921938896179, "median": 0.32308897376060486, "p05": 0.16778458654880524, "p95": 0.376542329788208 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3810515403747559, "group": 3, "latest_source_mass": 0.27817302942276, "layer": 10, "max_source_mass": 0.27817302942276, "mixer_index": 19, "normalized_entropy": 0.9962181042553622, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2385442852973938, "median": 0.23833481967449188, "p05": 0.2114236056804657, "p95": 0.2653840482234955 }, { "label": "completed_group_1", "mean": 0.23454181849956512, "median": 0.23530863225460052, "p05": 0.2103835791349411, "p95": 0.25546443462371826 }, { "label": "completed_group_2", "mean": 0.24874086678028107, "median": 0.24718257784843445, "p05": 0.22418582439422607, "p95": 0.2786770462989807 }, { "label": "current_group_partial", "mean": 0.27817302942276, "median": 0.28068703413009644, "p05": 0.22970353066921234, "p95": 0.3196732997894287 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.348030686378479, "group": 3, "latest_source_mass": 0.32103732228279114, "layer": 11, "max_source_mass": 0.32103732228279114, "mixer_index": 20, "normalized_entropy": 0.9723985931021886, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.23874624073505402, "median": 0.23695378005504608, "p05": 0.20177650451660156, "p95": 0.2892652153968811 }, { "label": "completed_group_1", "mean": 0.21300309896469116, "median": 0.186348557472229, "p05": 0.1455306112766266, "p95": 0.34363847970962524 }, { "label": "completed_group_2", "mean": 0.22721335291862488, "median": 0.22495108842849731, "p05": 0.20071068406105042, "p95": 0.2614164650440216 }, { "label": "current_group_partial", "mean": 0.32103732228279114, "median": 0.352428138256073, "p05": 0.1508321464061737, "p95": 0.4218788743019104 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3825427293777466, "group": 3, "latest_source_mass": 0.26641666889190674, "layer": 11, "max_source_mass": 0.26641666889190674, "mixer_index": 21, "normalized_entropy": 0.9972937697451836, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.24534732103347778, "median": 0.24462050199508667, "p05": 0.21986223757266998, "p95": 0.27343857288360596 }, { "label": "completed_group_1", "mean": 0.2375141680240631, "median": 0.2376786768436432, "p05": 0.20753934979438782, "p95": 0.27001917362213135 }, { "label": "completed_group_2", "mean": 0.25072187185287476, "median": 0.25058382749557495, "p05": 0.2306913286447525, "p95": 0.2714776396751404 }, { "label": "current_group_partial", "mean": 0.26641666889190674, "median": 0.2650451064109802, "p05": 0.22704681754112244, "p95": 0.31099480390548706 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3499886989593506, "group": 3, "latest_source_mass": 0.30865931510925293, "layer": 12, "max_source_mass": 0.30865931510925293, "mixer_index": 22, "normalized_entropy": 0.9738110006223994, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.24629786610603333, "median": 0.24514088034629822, "p05": 0.21092942357063293, "p95": 0.2848513722419739 }, { "label": "completed_group_1", "mean": 0.22150886058807373, "median": 0.19684141874313354, "p05": 0.15013524889945984, "p95": 0.34530556201934814 }, { "label": "completed_group_2", "mean": 0.2235339879989624, "median": 0.2229708731174469, "p05": 0.19156691431999207, "p95": 0.25894880294799805 }, { "label": "current_group_partial", "mean": 0.30865931510925293, "median": 0.34268343448638916, "p05": 0.13754047453403473, "p95": 0.424306720495224 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3807358741760254, "group": 3, "latest_source_mass": 0.2846421003341675, "layer": 12, "max_source_mass": 0.2846421003341675, "mixer_index": 23, "normalized_entropy": 0.9959903992256198, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.24433431029319763, "median": 0.24396280944347382, "p05": 0.22202064096927643, "p95": 0.26817846298217773 }, { "label": "completed_group_1", "mean": 0.23110222816467285, "median": 0.23034469783306122, "p05": 0.20346714556217194, "p95": 0.2573610544204712 }, { "label": "completed_group_2", "mean": 0.23992133140563965, "median": 0.23892486095428467, "p05": 0.2228689044713974, "p95": 0.2597067654132843 }, { "label": "current_group_partial", "mean": 0.2846421003341675, "median": 0.28294801712036133, "p05": 0.24798741936683655, "p95": 0.32731086015701294 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3399771451950073, "group": 4, "latest_source_mass": 0.3220147490501404, "layer": 13, "max_source_mass": 0.3220147490501404, "mixer_index": 24, "normalized_entropy": 0.9665891911386938, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24593281745910645, "median": 0.24438412487506866, "p05": 0.20850856602191925, "p95": 0.2918635606765747 }, { "label": "completed_group_1", "mean": 0.21928200125694275, "median": 0.1978234350681305, "p05": 0.14302152395248413, "p95": 0.34290891885757446 }, { "label": "completed_group_2", "mean": 0.21277041733264923, "median": 0.2074536681175232, "p05": 0.17290188372135162, "p95": 0.267916738986969 }, { "label": "completed_group_3", "mean": 0.3220147490501404, "median": 0.3550489544868469, "p05": 0.1336003690958023, "p95": 0.4569035768508911 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6075894832611084, "group": 4, "latest_source_mass": 0.20682592689990997, "layer": 13, "max_source_mass": 0.21128275990486145, "mixer_index": 25, "normalized_entropy": 0.998851506380761, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.19650748372077942, "median": 0.19601407647132874, "p05": 0.18472915887832642, "p95": 0.20989641547203064 }, { "label": "completed_group_1", "mean": 0.1956745684146881, "median": 0.1953330934047699, "p05": 0.18065369129180908, "p95": 0.2107330560684204 }, { "label": "completed_group_2", "mean": 0.18970930576324463, "median": 0.1893819272518158, "p05": 0.17753958702087402, "p95": 0.20207379758358002 }, { "label": "completed_group_3", "mean": 0.21128275990486145, "median": 0.21213039755821228, "p05": 0.19407641887664795, "p95": 0.22417660057544708 }, { "label": "current_group_partial", "mean": 0.20682592689990997, "median": 0.20550638437271118, "p05": 0.1884995847940445, "p95": 0.22830142080783844 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5706448554992676, "group": 4, "latest_source_mass": 0.2440074235200882, "layer": 14, "max_source_mass": 0.2440074235200882, "mixer_index": 26, "normalized_entropy": 0.9758965185080284, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1975565403699875, "median": 0.19368210434913635, "p05": 0.17189300060272217, "p95": 0.23318083584308624 }, { "label": "completed_group_1", "mean": 0.1772933304309845, "median": 0.15165507793426514, "p05": 0.12244623899459839, "p95": 0.29638439416885376 }, { "label": "completed_group_2", "mean": 0.16386015713214874, "median": 0.1534792184829712, "p05": 0.1269839107990265, "p95": 0.23500467836856842 }, { "label": "completed_group_3", "mean": 0.21728253364562988, "median": 0.21826648712158203, "p05": 0.14061616361141205, "p95": 0.29020458459854126 }, { "label": "current_group_partial", "mean": 0.2440074235200882, "median": 0.258046954870224, "p05": 0.10537702590227127, "p95": 0.330468088388443 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6042816638946533, "group": 4, "latest_source_mass": 0.2172817587852478, "layer": 14, "max_source_mass": 0.2172817587852478, "mixer_index": 27, "normalized_entropy": 0.9967962426511696, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21478912234306335, "median": 0.21512055397033691, "p05": 0.19715391099452972, "p95": 0.2305220067501068 }, { "label": "completed_group_1", "mean": 0.18552109599113464, "median": 0.18529027700424194, "p05": 0.1659296154975891, "p95": 0.20769654214382172 }, { "label": "completed_group_2", "mean": 0.17930489778518677, "median": 0.17780452966690063, "p05": 0.16197741031646729, "p95": 0.20606350898742676 }, { "label": "completed_group_3", "mean": 0.20310312509536743, "median": 0.20250298082828522, "p05": 0.19248157739639282, "p95": 0.21611490845680237 }, { "label": "current_group_partial", "mean": 0.2172817587852478, "median": 0.21857355535030365, "p05": 0.17562012374401093, "p95": 0.24863092601299286 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5764570236206055, "group": 4, "latest_source_mass": 0.25344932079315186, "layer": 15, "max_source_mass": 0.25344932079315186, "mixer_index": 28, "normalized_entropy": 0.9795078216073494, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20157726109027863, "median": 0.20011398196220398, "p05": 0.17114980518817902, "p95": 0.23868146538734436 }, { "label": "completed_group_1", "mean": 0.17631688714027405, "median": 0.156215101480484, "p05": 0.1293901652097702, "p95": 0.2730254530906677 }, { "label": "completed_group_2", "mean": 0.1703747808933258, "median": 0.1609533429145813, "p05": 0.1380072385072708, "p95": 0.2378104031085968 }, { "label": "completed_group_3", "mean": 0.19828173518180847, "median": 0.19690614938735962, "p05": 0.15977059304714203, "p95": 0.2402237504720688 }, { "label": "current_group_partial", "mean": 0.25344932079315186, "median": 0.27549347281455994, "p05": 0.10519556701183319, "p95": 0.33760181069374084 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6011263132095337, "group": 4, "latest_source_mass": 0.22829055786132812, "layer": 15, "max_source_mass": 0.22829055786132812, "mixer_index": 29, "normalized_entropy": 0.9948357130397182, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20813238620758057, "median": 0.20579253137111664, "p05": 0.18812841176986694, "p95": 0.23387062549591064 }, { "label": "completed_group_1", "mean": 0.18490281701087952, "median": 0.1840219497680664, "p05": 0.16537097096443176, "p95": 0.20639142394065857 }, { "label": "completed_group_2", "mean": 0.17333751916885376, "median": 0.16917650401592255, "p05": 0.15315541625022888, "p95": 0.2153732180595398 }, { "label": "completed_group_3", "mean": 0.20533671975135803, "median": 0.20556354522705078, "p05": 0.18881049752235413, "p95": 0.21959951519966125 }, { "label": "current_group_partial", "mean": 0.22829055786132812, "median": 0.23129136860370636, "p05": 0.17272692918777466, "p95": 0.2663574516773224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5817476511001587, "group": 4, "latest_source_mass": 0.24933090806007385, "layer": 16, "max_source_mass": 0.24933090806007385, "mixer_index": 30, "normalized_entropy": 0.9827950732861368, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20727969706058502, "median": 0.20713281631469727, "p05": 0.17457211017608643, "p95": 0.2406192123889923 }, { "label": "completed_group_1", "mean": 0.18177060782909393, "median": 0.1663498431444168, "p05": 0.14110450446605682, "p95": 0.2629965543746948 }, { "label": "completed_group_2", "mean": 0.17236655950546265, "median": 0.16402167081832886, "p05": 0.14090250432491302, "p95": 0.23658250272274017 }, { "label": "completed_group_3", "mean": 0.18925222754478455, "median": 0.18786820769309998, "p05": 0.16428329050540924, "p95": 0.21854622662067413 }, { "label": "current_group_partial", "mean": 0.24933090806007385, "median": 0.267303466796875, "p05": 0.10528035461902618, "p95": 0.33821040391921997 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.603710412979126, "group": 4, "latest_source_mass": 0.22391749918460846, "layer": 16, "max_source_mass": 0.22391749918460846, "mixer_index": 31, "normalized_entropy": 0.9964413045009534, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2016504406929016, "median": 0.1985834836959839, "p05": 0.18258583545684814, "p95": 0.2268678992986679 }, { "label": "completed_group_1", "mean": 0.1979413628578186, "median": 0.19931256771087646, "p05": 0.1786929965019226, "p95": 0.2149808406829834 }, { "label": "completed_group_2", "mean": 0.17889465391635895, "median": 0.17638051509857178, "p05": 0.15777523815631866, "p95": 0.20938827097415924 }, { "label": "completed_group_3", "mean": 0.19759604334831238, "median": 0.19828101992607117, "p05": 0.1828460842370987, "p95": 0.21084150671958923 }, { "label": "current_group_partial", "mean": 0.22391749918460846, "median": 0.22658780217170715, "p05": 0.17493478953838348, "p95": 0.26284241676330566 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5885388851165771, "group": 5, "latest_source_mass": 0.238150954246521, "layer": 17, "max_source_mass": 0.238150954246521, "mixer_index": 32, "normalized_entropy": 0.9870147042293073, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20988407731056213, "median": 0.2089688777923584, "p05": 0.18096143007278442, "p95": 0.24111014604568481 }, { "label": "completed_group_1", "mean": 0.18461555242538452, "median": 0.17398875951766968, "p05": 0.1473308950662613, "p95": 0.2504626214504242 }, { "label": "completed_group_2", "mean": 0.17252586781978607, "median": 0.1655999720096588, "p05": 0.14288564026355743, "p95": 0.23261940479278564 }, { "label": "completed_group_3", "mean": 0.19482356309890747, "median": 0.19468602538108826, "p05": 0.17603687942028046, "p95": 0.21431377530097961 }, { "label": "completed_group_4", "mean": 0.238150954246521, "median": 0.251210480928421, "p05": 0.11023568361997604, "p95": 0.3212902545928955 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.783994197845459, "group": 5, "latest_source_mass": 0.19404590129852295, "layer": 17, "max_source_mass": 0.19404590129852295, "mixer_index": 33, "normalized_entropy": 0.9956661195233188, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16140902042388916, "median": 0.1617078185081482, "p05": 0.13892026245594025, "p95": 0.18873091042041779 }, { "label": "completed_group_1", "mean": 0.16211673617362976, "median": 0.16129055619239807, "p05": 0.14639844000339508, "p95": 0.18146981298923492 }, { "label": "completed_group_2", "mean": 0.1512581706047058, "median": 0.14888465404510498, "p05": 0.1341290920972824, "p95": 0.18040500581264496 }, { "label": "completed_group_3", "mean": 0.1585075855255127, "median": 0.15648186206817627, "p05": 0.14301685988903046, "p95": 0.18330132961273193 }, { "label": "completed_group_4", "mean": 0.17266258597373962, "median": 0.1729961335659027, "p05": 0.1528140753507614, "p95": 0.19015821814537048 }, { "label": "current_group_partial", "mean": 0.19404590129852295, "median": 0.19937825202941895, "p05": 0.1327856332063675, "p95": 0.2279757857322693 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.765566349029541, "group": 5, "latest_source_mass": 0.22537899017333984, "layer": 18, "max_source_mass": 0.22537899017333984, "mixer_index": 34, "normalized_entropy": 0.9853813412746752, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1630604863166809, "median": 0.16281193494796753, "p05": 0.1371261477470398, "p95": 0.19141645729541779 }, { "label": "completed_group_1", "mean": 0.1509958803653717, "median": 0.14023837447166443, "p05": 0.12240280210971832, "p95": 0.21479898691177368 }, { "label": "completed_group_2", "mean": 0.14303798973560333, "median": 0.13656893372535706, "p05": 0.12068455666303635, "p95": 0.19587433338165283 }, { "label": "completed_group_3", "mean": 0.14991191029548645, "median": 0.1466745138168335, "p05": 0.1285857856273651, "p95": 0.18114891648292542 }, { "label": "completed_group_4", "mean": 0.16761471331119537, "median": 0.17043620347976685, "p05": 0.11439520120620728, "p95": 0.20673808455467224 }, { "label": "current_group_partial", "mean": 0.22537899017333984, "median": 0.2385822832584381, "p05": 0.11417834460735321, "p95": 0.2751238942146301 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7848252058029175, "group": 5, "latest_source_mass": 0.19710269570350647, "layer": 18, "max_source_mass": 0.19710269570350647, "mixer_index": 35, "normalized_entropy": 0.9961299138951252, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16126418113708496, "median": 0.16255977749824524, "p05": 0.1384611874818802, "p95": 0.1816936731338501 }, { "label": "completed_group_1", "mean": 0.1664336919784546, "median": 0.1663765162229538, "p05": 0.1537502110004425, "p95": 0.1792917549610138 }, { "label": "completed_group_2", "mean": 0.15361233055591583, "median": 0.15213868021965027, "p05": 0.13601605594158173, "p95": 0.17843274772167206 }, { "label": "completed_group_3", "mean": 0.1546509563922882, "median": 0.15342193841934204, "p05": 0.13999418914318085, "p95": 0.17627598345279694 }, { "label": "completed_group_4", "mean": 0.16693615913391113, "median": 0.16708451509475708, "p05": 0.1510891318321228, "p95": 0.18256354331970215 }, { "label": "current_group_partial", "mean": 0.19710269570350647, "median": 0.20123803615570068, "p05": 0.14279718697071075, "p95": 0.22455936670303345 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7731435298919678, "group": 5, "latest_source_mass": 0.2158534973859787, "layer": 19, "max_source_mass": 0.2158534973859787, "mixer_index": 36, "normalized_entropy": 0.9896102464332964, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.16123875975608826, "median": 0.16278928518295288, "p05": 0.131410151720047, "p95": 0.18839234113693237 }, { "label": "completed_group_1", "mean": 0.15981441736221313, "median": 0.15127545595169067, "p05": 0.13407617807388306, "p95": 0.21254204213619232 }, { "label": "completed_group_2", "mean": 0.1509810984134674, "median": 0.1480858325958252, "p05": 0.12804624438285828, "p95": 0.18798498809337616 }, { "label": "completed_group_3", "mean": 0.14913737773895264, "median": 0.14716270565986633, "p05": 0.12724824249744415, "p95": 0.17841869592666626 }, { "label": "completed_group_4", "mean": 0.16297483444213867, "median": 0.16397103667259216, "p05": 0.13277193903923035, "p95": 0.18706153333187103 }, { "label": "current_group_partial", "mean": 0.2158534973859787, "median": 0.2266644686460495, "p05": 0.1148078441619873, "p95": 0.2620621919631958 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7815005779266357, "group": 5, "latest_source_mass": 0.19993525743484497, "layer": 19, "max_source_mass": 0.19993525743484497, "mixer_index": 37, "normalized_entropy": 0.9942744037480438, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1552722007036209, "median": 0.15699897706508636, "p05": 0.12162899225950241, "p95": 0.19721786677837372 }, { "label": "completed_group_1", "mean": 0.17254950106143951, "median": 0.17343363165855408, "p05": 0.15400823950767517, "p95": 0.18830114603042603 }, { "label": "completed_group_2", "mean": 0.15193141996860504, "median": 0.14969776570796967, "p05": 0.13268329203128815, "p95": 0.1832970529794693 }, { "label": "completed_group_3", "mean": 0.15522463619709015, "median": 0.1547158658504486, "p05": 0.13487689197063446, "p95": 0.17792025208473206 }, { "label": "completed_group_4", "mean": 0.1650869995355606, "median": 0.16515228152275085, "p05": 0.14664505422115326, "p95": 0.18368470668792725 }, { "label": "current_group_partial", "mean": 0.19993525743484497, "median": 0.20347005128860474, "p05": 0.14352381229400635, "p95": 0.2376205176115036 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7801456451416016, "group": 5, "latest_source_mass": 0.2048441767692566, "layer": 20, "max_source_mass": 0.2048441767692566, "mixer_index": 38, "normalized_entropy": 0.9935182013624535, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1661754846572876, "median": 0.17102378606796265, "p05": 0.1277879923582077, "p95": 0.1914365589618683 }, { "label": "completed_group_1", "mean": 0.16477233171463013, "median": 0.16166681051254272, "p05": 0.14654575288295746, "p95": 0.19397443532943726 }, { "label": "completed_group_2", "mean": 0.15300315618515015, "median": 0.14975905418395996, "p05": 0.13582628965377808, "p95": 0.1866467446088791 }, { "label": "completed_group_3", "mean": 0.15292593836784363, "median": 0.15278922021389008, "p05": 0.13439597189426422, "p95": 0.17322586476802826 }, { "label": "completed_group_4", "mean": 0.1582789272069931, "median": 0.15707442164421082, "p05": 0.1421792358160019, "p95": 0.17823022603988647 }, { "label": "current_group_partial", "mean": 0.2048441767692566, "median": 0.2113300859928131, "p05": 0.1285371482372284, "p95": 0.2448955774307251 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7743862867355347, "group": 5, "latest_source_mass": 0.20740380883216858, "layer": 20, "max_source_mass": 0.20740380883216858, "mixer_index": 39, "normalized_entropy": 0.9903038422339103, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15205824375152588, "median": 0.15292710065841675, "p05": 0.09222303330898285, "p95": 0.21958938241004944 }, { "label": "completed_group_1", "mean": 0.1771804541349411, "median": 0.176810160279274, "p05": 0.1583203673362732, "p95": 0.19582124054431915 }, { "label": "completed_group_2", "mean": 0.15399661660194397, "median": 0.15262216329574585, "p05": 0.13103288412094116, "p95": 0.18263062834739685 }, { "label": "completed_group_3", "mean": 0.15122172236442566, "median": 0.14942464232444763, "p05": 0.12676094472408295, "p95": 0.18100851774215698 }, { "label": "completed_group_4", "mean": 0.15813913941383362, "median": 0.1581244319677353, "p05": 0.1325223445892334, "p95": 0.18428882956504822 }, { "label": "current_group_partial", "mean": 0.20740380883216858, "median": 0.21059799194335938, "p05": 0.1396750807762146, "p95": 0.2541891634464264 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.781898856163025, "group": 6, "latest_source_mass": 0.19959214329719543, "layer": 21, "max_source_mass": 0.19959214329719543, "mixer_index": 40, "normalized_entropy": 0.9944966870640967, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17058727145195007, "median": 0.1770990937948227, "p05": 0.12852564454078674, "p95": 0.19631126523017883 }, { "label": "completed_group_1", "mean": 0.1644459068775177, "median": 0.16324234008789062, "p05": 0.1497141271829605, "p95": 0.1824047863483429 }, { "label": "completed_group_2", "mean": 0.15141303837299347, "median": 0.14833912253379822, "p05": 0.1360013633966446, "p95": 0.1818353235721588 }, { "label": "completed_group_3", "mean": 0.15486884117126465, "median": 0.15303462743759155, "p05": 0.13701528310775757, "p95": 0.17965000867843628 }, { "label": "completed_group_4", "mean": 0.15909278392791748, "median": 0.158164381980896, "p05": 0.14224998652935028, "p95": 0.17945128679275513 }, { "label": "completed_group_5", "mean": 0.19959214329719543, "median": 0.20485615730285645, "p05": 0.1373821198940277, "p95": 0.23710061609745026 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9389331340789795, "group": 6, "latest_source_mass": 0.15687686204910278, "layer": 21, "max_source_mass": 0.15703952312469482, "mixer_index": 41, "normalized_entropy": 0.9964145235689732, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12895603477954865, "median": 0.1290992647409439, "p05": 0.10673296451568604, "p95": 0.16424264013767242 }, { "label": "completed_group_1", "mean": 0.15703952312469482, "median": 0.1581842303276062, "p05": 0.13781607151031494, "p95": 0.1730874478816986 }, { "label": "completed_group_2", "mean": 0.13832643628120422, "median": 0.1382940709590912, "p05": 0.12425417453050613, "p95": 0.15265563130378723 }, { "label": "completed_group_3", "mean": 0.13419196009635925, "median": 0.13376033306121826, "p05": 0.12005055695772171, "p95": 0.15059247612953186 }, { "label": "completed_group_4", "mean": 0.1312965601682663, "median": 0.12827834486961365, "p05": 0.11234064400196075, "p95": 0.1575525403022766 }, { "label": "completed_group_5", "mean": 0.15331262350082397, "median": 0.15450122952461243, "p05": 0.12799498438835144, "p95": 0.1745292842388153 }, { "label": "current_group_partial", "mean": 0.15687686204910278, "median": 0.15919944643974304, "p05": 0.13485465943813324, "p95": 0.171228289604187 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.941878080368042, "group": 6, "latest_source_mass": 0.1529344916343689, "layer": 22, "max_source_mass": 0.1529344916343689, "mixer_index": 42, "normalized_entropy": 0.9979279265852903, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14007402956485748, "median": 0.1409616768360138, "p05": 0.12191025912761688, "p95": 0.15652990341186523 }, { "label": "completed_group_1", "mean": 0.15248695015907288, "median": 0.15473851561546326, "p05": 0.1278151273727417, "p95": 0.17166844010353088 }, { "label": "completed_group_2", "mean": 0.14026451110839844, "median": 0.13944703340530396, "p05": 0.12999942898750305, "p95": 0.1532590687274933 }, { "label": "completed_group_3", "mean": 0.13737857341766357, "median": 0.13697952032089233, "p05": 0.12482102960348129, "p95": 0.15122388303279877 }, { "label": "completed_group_4", "mean": 0.1305495798587799, "median": 0.12951749563217163, "p05": 0.11473267525434494, "p95": 0.14860308170318604 }, { "label": "completed_group_5", "mean": 0.14631186425685883, "median": 0.14591258764266968, "p05": 0.13049930334091187, "p95": 0.1627071499824524 }, { "label": "current_group_partial", "mean": 0.1529344916343689, "median": 0.15251776576042175, "p05": 0.13440527021884918, "p95": 0.17216354608535767 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.936349630355835, "group": 6, "latest_source_mass": 0.16187208890914917, "layer": 22, "max_source_mass": 0.16187208890914917, "mixer_index": 43, "normalized_entropy": 0.9950868652881432, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12599323689937592, "median": 0.12448512017726898, "p05": 0.09913606941699982, "p95": 0.1710376739501953 }, { "label": "completed_group_1", "mean": 0.16029544174671173, "median": 0.1607722043991089, "p05": 0.13700781762599945, "p95": 0.1817699670791626 }, { "label": "completed_group_2", "mean": 0.1399340182542801, "median": 0.13928979635238647, "p05": 0.12660659849643707, "p95": 0.1542026251554489 }, { "label": "completed_group_3", "mean": 0.1333811730146408, "median": 0.1331656277179718, "p05": 0.11817716062068939, "p95": 0.15023785829544067 }, { "label": "completed_group_4", "mean": 0.12978437542915344, "median": 0.12865802645683289, "p05": 0.11021189391613007, "p95": 0.15201398730278015 }, { "label": "completed_group_5", "mean": 0.14873966574668884, "median": 0.14983436465263367, "p05": 0.12514030933380127, "p95": 0.16848590970039368 }, { "label": "current_group_partial", "mean": 0.16187208890914917, "median": 0.16017910838127136, "p05": 0.13439762592315674, "p95": 0.1932365447282791 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9385986328125, "group": 6, "latest_source_mass": 0.16527059674263, "layer": 23, "max_source_mass": 0.16527059674263, "mixer_index": 44, "normalized_entropy": 0.9962426239226088, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1397915780544281, "median": 0.14002375304698944, "p05": 0.12451328337192535, "p95": 0.15780359506607056 }, { "label": "completed_group_1", "mean": 0.15300613641738892, "median": 0.15646013617515564, "p05": 0.12481506168842316, "p95": 0.17462868988513947 }, { "label": "completed_group_2", "mean": 0.14345945417881012, "median": 0.14333182573318481, "p05": 0.12926152348518372, "p95": 0.158186674118042 }, { "label": "completed_group_3", "mean": 0.13018865883350372, "median": 0.1291605830192566, "p05": 0.11751946061849594, "p95": 0.14813899993896484 }, { "label": "completed_group_4", "mean": 0.1257915198802948, "median": 0.1252322494983673, "p05": 0.11126883327960968, "p95": 0.1417967826128006 }, { "label": "completed_group_5", "mean": 0.14249205589294434, "median": 0.1427975594997406, "p05": 0.12757454812526703, "p95": 0.15669940412044525 }, { "label": "current_group_partial", "mean": 0.16527059674263, "median": 0.16401387751102448, "p05": 0.13626056909561157, "p95": 0.20071014761924744 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.939380407333374, "group": 6, "latest_source_mass": 0.15541282296180725, "layer": 23, "max_source_mass": 0.15541282296180725, "mixer_index": 45, "normalized_entropy": 0.9966443765529929, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.128556489944458, "median": 0.12622612714767456, "p05": 0.1073562428355217, "p95": 0.16966179013252258 }, { "label": "completed_group_1", "mean": 0.15473899245262146, "median": 0.1559004783630371, "p05": 0.13596564531326294, "p95": 0.1694478690624237 }, { "label": "completed_group_2", "mean": 0.1372848004102707, "median": 0.13732792437076569, "p05": 0.12097055464982986, "p95": 0.15409716963768005 }, { "label": "completed_group_3", "mean": 0.13797816634178162, "median": 0.13845494389533997, "p05": 0.12182354182004929, "p95": 0.15258952975273132 }, { "label": "completed_group_4", "mean": 0.13512185215950012, "median": 0.1341967135667801, "p05": 0.11925841122865677, "p95": 0.15510126948356628 }, { "label": "completed_group_5", "mean": 0.15090686082839966, "median": 0.1500159651041031, "p05": 0.12717851996421814, "p95": 0.1754632294178009 }, { "label": "current_group_partial", "mean": 0.15541282296180725, "median": 0.15449640154838562, "p05": 0.13319969177246094, "p95": 0.17916548252105713 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9388402700424194, "group": 6, "latest_source_mass": 0.16304057836532593, "layer": 24, "max_source_mass": 0.16304057836532593, "mixer_index": 46, "normalized_entropy": 0.9963668008945192, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14282521605491638, "median": 0.14215797185897827, "p05": 0.12491188943386078, "p95": 0.16037693619728088 }, { "label": "completed_group_1", "mean": 0.15391701459884644, "median": 0.15735432505607605, "p05": 0.1199636161327362, "p95": 0.1801576018333435 }, { "label": "completed_group_2", "mean": 0.1462559700012207, "median": 0.14565740525722504, "p05": 0.13299958407878876, "p95": 0.1612643003463745 }, { "label": "completed_group_3", "mean": 0.13105326890945435, "median": 0.13004028797149658, "p05": 0.11590667068958282, "p95": 0.15094627439975739 }, { "label": "completed_group_4", "mean": 0.12675246596336365, "median": 0.1260158270597458, "p05": 0.11388987302780151, "p95": 0.1422860026359558 }, { "label": "completed_group_5", "mean": 0.13615547120571136, "median": 0.13547730445861816, "p05": 0.12139737606048584, "p95": 0.15295183658599854 }, { "label": "current_group_partial", "mean": 0.16304057836532593, "median": 0.16385850310325623, "p05": 0.13662296533584595, "p95": 0.18634286522865295 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9313902854919434, "group": 6, "latest_source_mass": 0.14112016558647156, "layer": 24, "max_source_mass": 0.17515115439891815, "mixer_index": 47, "normalized_entropy": 0.9925382661833493, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11694865673780441, "median": 0.11002877354621887, "p05": 0.09029232710599899, "p95": 0.1822434961795807 }, { "label": "completed_group_1", "mean": 0.17515115439891815, "median": 0.17486894130706787, "p05": 0.14681096374988556, "p95": 0.20170748233795166 }, { "label": "completed_group_2", "mean": 0.15024805068969727, "median": 0.14959201216697693, "p05": 0.13276571035385132, "p95": 0.17043110728263855 }, { "label": "completed_group_3", "mean": 0.1394161581993103, "median": 0.1390470266342163, "p05": 0.12232993543148041, "p95": 0.15828141570091248 }, { "label": "completed_group_4", "mean": 0.13194763660430908, "median": 0.13136030733585358, "p05": 0.11043333262205124, "p95": 0.15552270412445068 }, { "label": "completed_group_5", "mean": 0.1451682150363922, "median": 0.14562314748764038, "p05": 0.11801350116729736, "p95": 0.1739269196987152 }, { "label": "current_group_partial", "mean": 0.14112016558647156, "median": 0.137407124042511, "p05": 0.10931363701820374, "p95": 0.1778358519077301 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9366493225097656, "group": 7, "latest_source_mass": 0.16003848612308502, "layer": 25, "max_source_mass": 0.16076743602752686, "mixer_index": 48, "normalized_entropy": 0.9952408765892693, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14114168286323547, "median": 0.1450381577014923, "p05": 0.1118045300245285, "p95": 0.16293637454509735 }, { "label": "completed_group_1", "mean": 0.16076743602752686, "median": 0.1661323606967926, "p05": 0.12472015619277954, "p95": 0.18624728918075562 }, { "label": "completed_group_2", "mean": 0.14226865768432617, "median": 0.14061585068702698, "p05": 0.12671124935150146, "p95": 0.16414466500282288 }, { "label": "completed_group_3", "mean": 0.12737974524497986, "median": 0.127046599984169, "p05": 0.11332333087921143, "p95": 0.1442527025938034 }, { "label": "completed_group_4", "mean": 0.12711969017982483, "median": 0.12545642256736755, "p05": 0.11273720115423203, "p95": 0.14693106710910797 }, { "label": "completed_group_5", "mean": 0.1412842869758606, "median": 0.14015623927116394, "p05": 0.12225915491580963, "p95": 0.16319061815738678 }, { "label": "completed_group_6", "mean": 0.16003848612308502, "median": 0.1578693687915802, "p05": 0.12668105959892273, "p95": 0.19828300178050995 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.064785957336426, "group": 7, "latest_source_mass": 0.11369390040636063, "layer": 25, "max_source_mass": 0.15484434366226196, "mixer_index": 49, "normalized_entropy": 0.9929521537154775, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1061047688126564, "median": 0.09920068830251694, "p05": 0.08633361011743546, "p95": 0.15746362507343292 }, { "label": "completed_group_1", "mean": 0.15484434366226196, "median": 0.1575447916984558, "p05": 0.12716035544872284, "p95": 0.17626896500587463 }, { "label": "completed_group_2", "mean": 0.13487765192985535, "median": 0.1332845687866211, "p05": 0.11492810398340225, "p95": 0.15705637633800507 }, { "label": "completed_group_3", "mean": 0.12357068061828613, "median": 0.12331797182559967, "p05": 0.10956361889839172, "p95": 0.13827478885650635 }, { "label": "completed_group_4", "mean": 0.12023378908634186, "median": 0.11931760609149933, "p05": 0.10679896175861359, "p95": 0.13665954768657684 }, { "label": "completed_group_5", "mean": 0.12704278528690338, "median": 0.12507274746894836, "p05": 0.10655903816223145, "p95": 0.15118366479873657 }, { "label": "completed_group_6", "mean": 0.11963208019733429, "median": 0.11901399493217468, "p05": 0.09560881555080414, "p95": 0.14494498074054718 }, { "label": "current_group_partial", "mean": 0.11369390040636063, "median": 0.10261981189250946, "p05": 0.08823835849761963, "p95": 0.17856189608573914 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.062044143676758, "group": 7, "latest_source_mass": 0.12150939553976059, "layer": 26, "max_source_mass": 0.14862585067749023, "mixer_index": 50, "normalized_entropy": 0.9916336200588627, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11719765514135361, "median": 0.12396101653575897, "p05": 0.05985046178102493, "p95": 0.15798619389533997 }, { "label": "completed_group_1", "mean": 0.14862585067749023, "median": 0.15304353833198547, "p05": 0.10433162748813629, "p95": 0.18226584792137146 }, { "label": "completed_group_2", "mean": 0.12573553621768951, "median": 0.12414389848709106, "p05": 0.11025667190551758, "p95": 0.1469687968492508 }, { "label": "completed_group_3", "mean": 0.12179906666278839, "median": 0.1216571182012558, "p05": 0.10140006989240646, "p95": 0.14223159849643707 }, { "label": "completed_group_4", "mean": 0.11081404983997345, "median": 0.11002233624458313, "p05": 0.093814417719841, "p95": 0.13098381459712982 }, { "label": "completed_group_5", "mean": 0.1260838657617569, "median": 0.1261553317308426, "p05": 0.10700763761997223, "p95": 0.14507941901683807 }, { "label": "completed_group_6", "mean": 0.1282345950603485, "median": 0.12765415012836456, "p05": 0.09908819198608398, "p95": 0.15842299163341522 }, { "label": "current_group_partial", "mean": 0.12150939553976059, "median": 0.11118246614933014, "p05": 0.0877910628914833, "p95": 0.18945831060409546 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.054962158203125, "group": 7, "latest_source_mass": 0.09995201230049133, "layer": 26, "max_source_mass": 0.16456037759780884, "mixer_index": 51, "normalized_entropy": 0.9882279049513767, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10514847934246063, "median": 0.09734492748975754, "p05": 0.08340487629175186, "p95": 0.16154851019382477 }, { "label": "completed_group_1", "mean": 0.16456037759780884, "median": 0.16979515552520752, "p05": 0.12305554002523422, "p95": 0.1964556723833084 }, { "label": "completed_group_2", "mean": 0.14191412925720215, "median": 0.141158789396286, "p05": 0.11155636608600616, "p95": 0.17160338163375854 }, { "label": "completed_group_3", "mean": 0.1262327879667282, "median": 0.12533226609230042, "p05": 0.11042644083499908, "p95": 0.14479395747184753 }, { "label": "completed_group_4", "mean": 0.1181877925992012, "median": 0.11585173010826111, "p05": 0.10073923319578171, "p95": 0.14371660351753235 }, { "label": "completed_group_5", "mean": 0.12522782385349274, "median": 0.12385813146829605, "p05": 0.10391169786453247, "p95": 0.14975282549858093 }, { "label": "completed_group_6", "mean": 0.11877657473087311, "median": 0.11736974120140076, "p05": 0.09251725673675537, "p95": 0.1482948511838913 }, { "label": "current_group_partial", "mean": 0.09995201230049133, "median": 0.08531178534030914, "p05": 0.07465144991874695, "p95": 0.17865940928459167 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0610406398773193, "group": 7, "latest_source_mass": 0.10865749418735504, "layer": 27, "max_source_mass": 0.15103808045387268, "mixer_index": 52, "normalized_entropy": 0.9911510367405416, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11198848485946655, "median": 0.11365711688995361, "p05": 0.06384186446666718, "p95": 0.16414028406143188 }, { "label": "completed_group_1", "mean": 0.15103808045387268, "median": 0.15550360083580017, "p05": 0.10918902605772018, "p95": 0.18123289942741394 }, { "label": "completed_group_2", "mean": 0.13588255643844604, "median": 0.1351599097251892, "p05": 0.118770532310009, "p95": 0.1558580845594406 }, { "label": "completed_group_3", "mean": 0.12388212233781815, "median": 0.12342942506074905, "p05": 0.10572439432144165, "p95": 0.14399279654026031 }, { "label": "completed_group_4", "mean": 0.11551117897033691, "median": 0.11412014812231064, "p05": 0.09702129662036896, "p95": 0.13879118859767914 }, { "label": "completed_group_5", "mean": 0.12874530255794525, "median": 0.12885355949401855, "p05": 0.10612692683935165, "p95": 0.15165534615516663 }, { "label": "completed_group_6", "mean": 0.12429477274417877, "median": 0.12331919372081757, "p05": 0.09724370390176773, "p95": 0.1537407487630844 }, { "label": "current_group_partial", "mean": 0.10865749418735504, "median": 0.0960599035024643, "p05": 0.08229582756757736, "p95": 0.18050634860992432 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0338308811187744, "group": 7, "latest_source_mass": 0.08955829590559006, "layer": 27, "max_source_mass": 0.17635446786880493, "mixer_index": 53, "normalized_entropy": 0.9780659087322957, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.0961676836013794, "median": 0.08641783893108368, "p05": 0.06882427632808685, "p95": 0.17455565929412842 }, { "label": "completed_group_1", "mean": 0.17635446786880493, "median": 0.1809459924697876, "p05": 0.12560009956359863, "p95": 0.21768227219581604 }, { "label": "completed_group_2", "mean": 0.14848342537879944, "median": 0.14817440509796143, "p05": 0.1071014404296875, "p95": 0.18990638852119446 }, { "label": "completed_group_3", "mean": 0.13749773800373077, "median": 0.13687211275100708, "p05": 0.11250527948141098, "p95": 0.16500277817249298 }, { "label": "completed_group_4", "mean": 0.12027013301849365, "median": 0.11871654540300369, "p05": 0.09309259057044983, "p95": 0.15373988449573517 }, { "label": "completed_group_5", "mean": 0.12292921543121338, "median": 0.1185426190495491, "p05": 0.09395024180412292, "p95": 0.1612403392791748 }, { "label": "completed_group_6", "mean": 0.10873906314373016, "median": 0.1079384982585907, "p05": 0.07837513834238052, "p95": 0.14415065944194794 }, { "label": "current_group_partial", "mean": 0.08955829590559006, "median": 0.07020936906337738, "p05": 0.059403665363788605, "p95": 0.19719363749027252 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.068699598312378, "group": 7, "latest_source_mass": 0.1132618635892868, "layer": 28, "max_source_mass": 0.14292436838150024, "mixer_index": 54, "normalized_entropy": 0.9948342171914195, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11418520659208298, "median": 0.10965901613235474, "p05": 0.09243083000183105, "p95": 0.14435738325119019 }, { "label": "completed_group_1", "mean": 0.14292436838150024, "median": 0.14914286136627197, "p05": 0.10227109491825104, "p95": 0.16936004161834717 }, { "label": "completed_group_2", "mean": 0.1359846293926239, "median": 0.1383761763572693, "p05": 0.10866174101829529, "p95": 0.15511028468608856 }, { "label": "completed_group_3", "mean": 0.13056153059005737, "median": 0.1301957666873932, "p05": 0.11531321704387665, "p95": 0.1473139524459839 }, { "label": "completed_group_4", "mean": 0.12100572884082794, "median": 0.11918226629495621, "p05": 0.1037902981042862, "p95": 0.14383943378925323 }, { "label": "completed_group_5", "mean": 0.11985942721366882, "median": 0.11902119219303131, "p05": 0.10603872686624527, "p95": 0.1366841346025467 }, { "label": "completed_group_6", "mean": 0.12221725285053253, "median": 0.12153760343790054, "p05": 0.10227984189987183, "p95": 0.14314256608486176 }, { "label": "current_group_partial", "mean": 0.1132618635892868, "median": 0.10621161758899689, "p05": 0.09176099300384521, "p95": 0.16215074062347412 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.036879539489746, "group": 7, "latest_source_mass": 0.08889776468276978, "layer": 28, "max_source_mass": 0.17786666750907898, "mixer_index": 55, "normalized_entropy": 0.9795320035033508, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09746268391609192, "median": 0.08779538422822952, "p05": 0.0718376636505127, "p95": 0.17390751838684082 }, { "label": "completed_group_1", "mean": 0.17786666750907898, "median": 0.18242985010147095, "p05": 0.12422755360603333, "p95": 0.21792855858802795 }, { "label": "completed_group_2", "mean": 0.15074898302555084, "median": 0.15042778849601746, "p05": 0.10790883004665375, "p95": 0.19122514128684998 }, { "label": "completed_group_3", "mean": 0.13200223445892334, "median": 0.13159450888633728, "p05": 0.1103091835975647, "p95": 0.15569928288459778 }, { "label": "completed_group_4", "mean": 0.11842608451843262, "median": 0.11631985008716583, "p05": 0.09421451389789581, "p95": 0.1504896730184555 }, { "label": "completed_group_5", "mean": 0.12164927273988724, "median": 0.11936528235673904, "p05": 0.09542038291692734, "p95": 0.15253053605556488 }, { "label": "completed_group_6", "mean": 0.1129462867975235, "median": 0.11048559844493866, "p05": 0.08038562536239624, "p95": 0.15222208201885223 }, { "label": "current_group_partial", "mean": 0.08889776468276978, "median": 0.07258449494838715, "p05": 0.06192244961857796, "p95": 0.18029186129570007 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.06752347946167, "group": 8, "latest_source_mass": 0.1112038791179657, "layer": 29, "max_source_mass": 0.14705273509025574, "mixer_index": 56, "normalized_entropy": 0.994268623580282, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11505837738513947, "median": 0.11106020212173462, "p05": 0.08959393203258514, "p95": 0.14650048315525055 }, { "label": "completed_group_1", "mean": 0.14705273509025574, "median": 0.1517227590084076, "p05": 0.10864706337451935, "p95": 0.17754200100898743 }, { "label": "completed_group_2", "mean": 0.1377231776714325, "median": 0.13775423169136047, "p05": 0.11708228290081024, "p95": 0.1583956629037857 }, { "label": "completed_group_3", "mean": 0.12483140826225281, "median": 0.12468798458576202, "p05": 0.10894858092069626, "p95": 0.1416374146938324 }, { "label": "completed_group_4", "mean": 0.11869609355926514, "median": 0.11688373982906342, "p05": 0.10011905431747437, "p95": 0.14243903756141663 }, { "label": "completed_group_5", "mean": 0.1215740367770195, "median": 0.12072279304265976, "p05": 0.1052989736199379, "p95": 0.13997763395309448 }, { "label": "completed_group_6", "mean": 0.12386028468608856, "median": 0.12289520353078842, "p05": 0.10213380306959152, "p95": 0.14759738743305206 }, { "label": "completed_group_7", "mean": 0.1112038791179657, "median": 0.10434558987617493, "p05": 0.09146858006715775, "p95": 0.15371212363243103 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1517844200134277, "group": 8, "latest_source_mass": 0.0917041003704071, "layer": 29, "max_source_mass": 0.16287443041801453, "mixer_index": 57, "normalized_entropy": 0.97931929317035, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08786985278129578, "median": 0.07908640056848526, "p05": 0.06481664627790451, "p95": 0.1606782078742981 }, { "label": "completed_group_1", "mean": 0.16287443041801453, "median": 0.16780570149421692, "p05": 0.11163335293531418, "p95": 0.20008432865142822 }, { "label": "completed_group_2", "mean": 0.13886377215385437, "median": 0.13884207606315613, "p05": 0.09857554733753204, "p95": 0.17596080899238586 }, { "label": "completed_group_3", "mean": 0.12039706110954285, "median": 0.11977612972259521, "p05": 0.09957708418369293, "p95": 0.14385205507278442 }, { "label": "completed_group_4", "mean": 0.10935483872890472, "median": 0.10729463398456573, "p05": 0.08751945197582245, "p95": 0.1386163830757141 }, { "label": "completed_group_5", "mean": 0.1119438111782074, "median": 0.10884027928113937, "p05": 0.08701130002737045, "p95": 0.14368495345115662 }, { "label": "completed_group_6", "mean": 0.10067413747310638, "median": 0.09888175874948502, "p05": 0.07253709435462952, "p95": 0.13528487086296082 }, { "label": "completed_group_7", "mean": 0.07631799578666687, "median": 0.06552867591381073, "p05": 0.05597877502441406, "p95": 0.13834024965763092 }, { "label": "current_group_partial", "mean": 0.0917041003704071, "median": 0.08405104279518127, "p05": 0.06457734853029251, "p95": 0.149135559797287 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.177518129348755, "group": 8, "latest_source_mass": 0.09682197868824005, "layer": 30, "max_source_mass": 0.13366679847240448, "mixer_index": 58, "normalized_entropy": 0.9910312090121641, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10418819636106491, "median": 0.10777237266302109, "p05": 0.05029530078172684, "p95": 0.157025545835495 }, { "label": "completed_group_1", "mean": 0.13366679847240448, "median": 0.13715378940105438, "p05": 0.09650824964046478, "p95": 0.16189607977867126 }, { "label": "completed_group_2", "mean": 0.12014994770288467, "median": 0.1188649833202362, "p05": 0.10183564573526382, "p95": 0.14117459952831268 }, { "label": "completed_group_3", "mean": 0.11398349702358246, "median": 0.11387990415096283, "p05": 0.09552477300167084, "p95": 0.13391552865505219 }, { "label": "completed_group_4", "mean": 0.10732515156269073, "median": 0.10627084970474243, "p05": 0.08923279494047165, "p95": 0.12864264845848083 }, { "label": "completed_group_5", "mean": 0.12029910832643509, "median": 0.12015098333358765, "p05": 0.09821736067533493, "p95": 0.1429397165775299 }, { "label": "completed_group_6", "mean": 0.11035183072090149, "median": 0.1095365658402443, "p05": 0.0866188108921051, "p95": 0.1372661590576172 }, { "label": "completed_group_7", "mean": 0.09321349859237671, "median": 0.08730670809745789, "p05": 0.07530548423528671, "p95": 0.13010826706886292 }, { "label": "current_group_partial", "mean": 0.09682197868824005, "median": 0.08937598764896393, "p05": 0.07608069479465485, "p95": 0.14057159423828125 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1535391807556152, "group": 8, "latest_source_mass": 0.08107093721628189, "layer": 30, "max_source_mass": 0.1632661372423172, "mixer_index": 59, "normalized_entropy": 0.980117919200792, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08976081013679504, "median": 0.08317285776138306, "p05": 0.06678152084350586, "p95": 0.15141215920448303 }, { "label": "completed_group_1", "mean": 0.1632661372423172, "median": 0.16861677169799805, "p05": 0.10940256714820862, "p95": 0.20118498802185059 }, { "label": "completed_group_2", "mean": 0.13892531394958496, "median": 0.13932356238365173, "p05": 0.09704238921403885, "p95": 0.1767924726009369 }, { "label": "completed_group_3", "mean": 0.12143255770206451, "median": 0.12173299491405487, "p05": 0.09844215214252472, "p95": 0.14377053081989288 }, { "label": "completed_group_4", "mean": 0.10986582189798355, "median": 0.10805591940879822, "p05": 0.08902113884687424, "p95": 0.13708043098449707 }, { "label": "completed_group_5", "mean": 0.11029069125652313, "median": 0.10775443911552429, "p05": 0.08701202273368835, "p95": 0.13905951380729675 }, { "label": "completed_group_6", "mean": 0.10418520867824554, "median": 0.10218185186386108, "p05": 0.07623609900474548, "p95": 0.14061832427978516 }, { "label": "completed_group_7", "mean": 0.08120252937078476, "median": 0.06974737346172333, "p05": 0.059275463223457336, "p95": 0.14717024564743042 }, { "label": "current_group_partial", "mean": 0.08107093721628189, "median": 0.07502967119216919, "p05": 0.06442901492118835, "p95": 0.11666787415742874 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.191593647003174, "group": 8, "latest_source_mass": 0.10164810717105865, "layer": 31, "max_source_mass": 0.12338979542255402, "mixer_index": 60, "normalized_entropy": 0.9974372531642294, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10739251971244812, "median": 0.10964220017194748, "p05": 0.08550600707530975, "p95": 0.12537474930286407 }, { "label": "completed_group_1", "mean": 0.12338979542255402, "median": 0.1267070472240448, "p05": 0.09627094119787216, "p95": 0.13979919254779816 }, { "label": "completed_group_2", "mean": 0.11759915947914124, "median": 0.11828742921352386, "p05": 0.10300115495920181, "p95": 0.12881097197532654 }, { "label": "completed_group_3", "mean": 0.1066875234246254, "median": 0.10600249469280243, "p05": 0.09917435050010681, "p95": 0.11636147648096085 }, { "label": "completed_group_4", "mean": 0.1060015857219696, "median": 0.10592096298933029, "p05": 0.0979120284318924, "p95": 0.11414148658514023 }, { "label": "completed_group_5", "mean": 0.1124890148639679, "median": 0.11278317868709564, "p05": 0.101162388920784, "p95": 0.12259824573993683 }, { "label": "completed_group_6", "mean": 0.11740899085998535, "median": 0.11669919639825821, "p05": 0.10286425054073334, "p95": 0.13360337913036346 }, { "label": "completed_group_7", "mean": 0.10738331079483032, "median": 0.10325507819652557, "p05": 0.09494076669216156, "p95": 0.13414672017097473 }, { "label": "current_group_partial", "mean": 0.10164810717105865, "median": 0.09851109236478806, "p05": 0.08805661648511887, "p95": 0.12722453474998474 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1465134620666504, "group": 8, "latest_source_mass": 0.07361725717782974, "layer": 31, "max_source_mass": 0.165542334318161, "mixer_index": 61, "normalized_entropy": 0.9769203768278215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08895281702280045, "median": 0.08148864656686783, "p05": 0.0638735294342041, "p95": 0.16049745678901672 }, { "label": "completed_group_1", "mean": 0.165542334318161, "median": 0.1687086820602417, "p05": 0.11728888005018234, "p95": 0.20459574460983276 }, { "label": "completed_group_2", "mean": 0.14156755805015564, "median": 0.14117418229579926, "p05": 0.09794186800718307, "p95": 0.18480409681797028 }, { "label": "completed_group_3", "mean": 0.12290289998054504, "median": 0.12261313199996948, "p05": 0.09803362190723419, "p95": 0.14901992678642273 }, { "label": "completed_group_4", "mean": 0.11169517040252686, "median": 0.10987502336502075, "p05": 0.09000048041343689, "p95": 0.14066611230373383 }, { "label": "completed_group_5", "mean": 0.11284246295690536, "median": 0.11086639761924744, "p05": 0.08716568350791931, "p95": 0.14374534785747528 }, { "label": "completed_group_6", "mean": 0.10374521464109421, "median": 0.10118988156318665, "p05": 0.07450100779533386, "p95": 0.14089089632034302 }, { "label": "completed_group_7", "mean": 0.0791342556476593, "median": 0.06692048162221909, "p05": 0.055950865149497986, "p95": 0.15064169466495514 }, { "label": "current_group_partial", "mean": 0.07361725717782974, "median": 0.06859230995178223, "p05": 0.05816151574254036, "p95": 0.10351020097732544 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1857833862304688, "group": 8, "latest_source_mass": 0.09919881820678711, "layer": 32, "max_source_mass": 0.12139561772346497, "mixer_index": 62, "normalized_entropy": 0.9947928895281057, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10998444259166718, "median": 0.11795933544635773, "p05": 0.05861174687743187, "p95": 0.14438092708587646 }, { "label": "completed_group_1", "mean": 0.12038751691579819, "median": 0.12215788662433624, "p05": 0.09455405175685883, "p95": 0.141767680644989 }, { "label": "completed_group_2", "mean": 0.11211743205785751, "median": 0.111464723944664, "p05": 0.09543715417385101, "p95": 0.130376935005188 }, { "label": "completed_group_3", "mean": 0.11047163605690002, "median": 0.11037711799144745, "p05": 0.09404903650283813, "p95": 0.127632275223732 }, { "label": "completed_group_4", "mean": 0.10957512259483337, "median": 0.10831746459007263, "p05": 0.09309716522693634, "p95": 0.12977787852287292 }, { "label": "completed_group_5", "mean": 0.12139561772346497, "median": 0.12195655703544617, "p05": 0.10015495121479034, "p95": 0.1401825100183487 }, { "label": "completed_group_6", "mean": 0.11392621695995331, "median": 0.11355376243591309, "p05": 0.09452947974205017, "p95": 0.1352669596672058 }, { "label": "completed_group_7", "mean": 0.10294321179389954, "median": 0.09949207305908203, "p05": 0.08735473453998566, "p95": 0.13111063838005066 }, { "label": "current_group_partial", "mean": 0.09919881820678711, "median": 0.09456383436918259, "p05": 0.0812632143497467, "p95": 0.13172483444213867 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1400556564331055, "group": 8, "latest_source_mass": 0.0691327303647995, "layer": 32, "max_source_mass": 0.16802933812141418, "mixer_index": 63, "normalized_entropy": 0.9739813028250293, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08778932690620422, "median": 0.0795741081237793, "p05": 0.06344711035490036, "p95": 0.16418468952178955 }, { "label": "completed_group_1", "mean": 0.16802933812141418, "median": 0.16981908679008484, "p05": 0.12191798537969589, "p95": 0.20580793917179108 }, { "label": "completed_group_2", "mean": 0.14664983749389648, "median": 0.14726951718330383, "p05": 0.09816566854715347, "p95": 0.1929507553577423 }, { "label": "completed_group_3", "mean": 0.12668848037719727, "median": 0.1260564625263214, "p05": 0.10192481428384781, "p95": 0.1541551649570465 }, { "label": "completed_group_4", "mean": 0.11206841468811035, "median": 0.10879923403263092, "p05": 0.08885316550731659, "p95": 0.1473042368888855 }, { "label": "completed_group_5", "mean": 0.11208956688642502, "median": 0.10941921174526215, "p05": 0.08551695197820663, "p95": 0.1449490785598755 }, { "label": "completed_group_6", "mean": 0.10068599134683609, "median": 0.09787309914827347, "p05": 0.07257338613271713, "p95": 0.1365347057580948 }, { "label": "completed_group_7", "mean": 0.07686632871627808, "median": 0.06485281884670258, "p05": 0.05413344129920006, "p95": 0.14826452732086182 }, { "label": "current_group_partial", "mean": 0.0691327303647995, "median": 0.06579415500164032, "p05": 0.055686283856630325, "p95": 0.0925217717885971 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1613364219665527, "group": null, "latest_source_mass": 0.12122741341590881, "layer": null, "max_source_mass": 0.15537577867507935, "mixer_index": null, "normalized_entropy": 0.9836665966056254, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1255332976579666, "median": 0.12688711285591125, "p05": 0.08593897521495819, "p95": 0.16157466173171997 }, { "label": "completed_group_1", "mean": 0.08712290227413177, "median": 0.08774389326572418, "p05": 0.06929010152816772, "p95": 0.10448890924453735 }, { "label": "completed_group_2", "mean": 0.09080197662115097, "median": 0.09027823060750961, "p05": 0.07495184242725372, "p95": 0.10968034714460373 }, { "label": "completed_group_3", "mean": 0.08838342130184174, "median": 0.0878291055560112, "p05": 0.07189641892910004, "p95": 0.10652950406074524 }, { "label": "completed_group_4", "mean": 0.09530800580978394, "median": 0.09452416002750397, "p05": 0.07556769251823425, "p95": 0.11846894770860672 }, { "label": "completed_group_5", "mean": 0.09807588160037994, "median": 0.0959044098854065, "p05": 0.07471995055675507, "p95": 0.12936776876449585 }, { "label": "completed_group_6", "mean": 0.13817133009433746, "median": 0.13675633072853088, "p05": 0.10469131171703339, "p95": 0.17332828044891357 }, { "label": "completed_group_7", "mean": 0.15537577867507935, "median": 0.1563301980495453, "p05": 0.12374594062566757, "p95": 0.18756605684757233 }, { "label": "completed_group_8", "mean": 0.12122741341590881, "median": 0.12231454998254776, "p05": 0.06295841932296753, "p95": 0.17872877418994904 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "83aa24720e9c1fc671d5681102b923705e3423225e358f5b850e1a9ed514d9b6", "optimizer_state_before": "83aa24720e9c1fc671d5681102b923705e3423225e358f5b850e1a9ed514d9b6", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "fe2bfa0ef8e4259f3ed7359d56fb5bbe56e2b0eafc30ccf1c1e3233cb3ec1708", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.01049753837287426, 0.013886291533708572, 0.01245343592017889, 0.01395952794700861, 0.014544809237122536, 0.014345151372253895, 0.015135552734136581, 0.014776745811104774, 0.01795734278857708, 0.013709688559174538, 0.016144096851348877, 0.018440017476677895, 0.018456056714057922, 0.012218938209116459, 0.01701919361948967, 0.01305371057242155, 0.017418840900063515, 0.026864025741815567, 0.035265542566776276, 0.027374446392059326, 0.01969025284051895, 0.018602686002850533, 0.0133047541603446, 0.015662170946598053, 0.012774706818163395, 0.016374029219150543, 0.010044055990874767, 0.006929588038474321, 0.005331806372851133, 0.01223068032413721, 0.00525518087670207, 0.010387555696070194 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.7470343841050635e-05, "first_to_last_ratio": 5.543186856446801, "imbalance_abs_log_ratio": 1.7125695802152925, "last_quartile_mean": 6.759711482118291e-06, "mean": 1.9353574231217863e-05, "normalized": [ 2.3427102645191087, 2.192158088708691, 2.1504138353757374, 2.019465522152087, 1.9995945477738173, 1.7731063055219731, 1.5680823096714636, 1.4432231490291196, 1.4193844085037457, 1.2773784002125157, 1.1654379763631353, 1.0792980891066979, 1.0342522513219847, 0.9566360686297143, 0.8835644518961069, 0.8115541392967873, 0.8495762151917138, 0.7730101521498506, 0.7009121964310462, 0.6382247620634326, 0.6098072442169371, 0.5543251081308368, 0.501441625782686, 0.46224625993377594, 0.4400253690998266, 0.4103016633144128, 0.39090671090870194, 0.3772935492489135, 0.30325680084430284, 0.29393754112208775, 0.2900082627577495, 0.28846673072104 ], "peak_layer": 1, "peak_normalized": 2.3427102645191087, "population_cv": 0.6284664540855158, "rest_mean": 2.109694017875612e-05, "spike_contrast": 0.4711298430148607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.939398114511278e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.533981700660661e-05, 4.242609429638833e-05, 4.161819379078224e-05, 3.908387589035556e-05, 3.8699301512679085e-05, 3.4315944503759965e-05, 3.0347997380886227e-05, 2.7931526346947066e-05, 2.7470161512610503e-05, 2.4721837689867243e-05, 2.2555390387424268e-05, 2.088827568513807e-05, 2.0016477719764225e-05, 1.85143271664856e-05, 1.7100130207836628e-05, 1.5706473277532496e-05, 1.6442336345789954e-05, 1.4960509361117147e-05, 1.3565156223194208e-05, 1.2351930308796e-05, 1.1801949767686892e-05, 1.0728172128438018e-05, 9.704687727207784e-06, 8.94611730473116e-06, 8.516063644492533e-06, 7.940803698147647e-06, 7.565442047052784e-06, 7.3019787123485e-06, 5.8691030062618665e-06, 5.688742021447979e-06, 5.612696440948639e-06, 5.582862286246382e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001431262517144205, "first_to_last_ratio": 5.283860582769421, "imbalance_abs_log_ratio": 1.6646570015219528, "last_quartile_mean": 2.7087439093520516e-05, "mean": 7.517440684523535e-05, "normalized": [ 2.3167821007513925, 2.1576627418693985, 2.0809366190077223, 1.9632050338348952, 1.9581380043895957, 1.7565617549117303, 1.5601488386363862, 1.4379456619222077, 1.4181936161344835, 1.2807776397732267, 1.1719405534721656, 1.0858056019237723, 1.04268310234457, 0.9685159493695125, 0.894804763228899, 0.8229797757020919, 0.8654572820771477, 0.7870405201408273, 0.7143484600420894, 0.6515578296073351, 0.6246802736621585, 0.5677900631265306, 0.5145973361420437, 0.47482296492635234, 0.451578528714676, 0.4221160637804328, 0.4032803071079029, 0.38978326294250304, 0.3131724384233296, 0.3039367871211443, 0.2999847761272356, 0.29877134878624156 ], "peak_layer": 1, "peak_normalized": 2.3167821007513925, "population_cv": 0.6100414888257124, "rest_mean": 8.17633902409364e-05, "spike_contrast": 0.48424969160129383, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.959389650844969e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001741627202136442, 0.0001622010167920962, 0.00015643317601643503, 0.00014758277393411845, 0.00014720186300110072, 0.000132048488012515, 0.00011728326353477314, 0.00010809671221068129, 0.00010661186388460919, 9.628169937059283e-05, 8.809993596514687e-05, 8.162479207385331e-05, 7.838308374630287e-05, 7.28076120140031e-05, 6.726641731802374e-05, 6.186701648402959e-05, 6.506023783003911e-05, 5.9165304264752194e-05, 5.370072176447138e-05, 4.898047336610034e-05, 4.695996904047206e-05, 4.268328120815568e-05, 3.868454950861633e-05, 3.569453474483453e-05, 3.394714804016985e-05, 3.173232471453957e-05, 3.031635787920095e-05, 2.930172558990307e-05, 2.3542552298749797e-05, 2.284826769027859e-05, 2.2551177607965656e-05, 2.2459958927356638e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12692579254508018, "first_to_last_ratio": 5.283860704622959, "imbalance_abs_log_ratio": 1.6646570245834122, "last_quartile_mean": 0.02402141154743731, "mean": 0.06666541617596522, "normalized": [ 2.316782047314218, 2.1576628243019496, 2.0809365727315674, 1.9632050581780958, 1.9581380467877862, 1.7565616761708855, 1.5601488782800663, 1.4379457021460735, 1.4181936562228672, 1.2807775992725852, 1.1719405684082436, 1.085805621683585, 1.042683162005052, 0.9685160093542362, 0.8948047849086074, 0.8229797475738845, 0.8654572915254822, 0.7870405460142597, 0.714348412471682, 0.651557783354314, 0.6246802596485354, 0.5677900422465361, 0.5145973193268033, 0.4748229339727928, 0.4515785025852313, 0.4221160632138638, 0.40328031619304167, 0.38978324313999146, 0.31317244048147536, 0.3039367789939916, 0.29998477721787487, 0.2987713342744216 ], "peak_layer": 1, "peak_normalized": 2.316782047314218, "population_cv": 0.6100414954327734, "rest_mean": 0.07250859284842456, "spike_contrast": 0.48424966980237977, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03511226214468479, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.1544492393732071, 0.14384149014949799, 0.13872650265693665, 0.13087788224220276, 0.13054008781909943, 0.11710191518068314, 0.10400797426700592, 0.09586124867200851, 0.09454447031021118, 0.0853835716843605, 0.0781279057264328, 0.07238568365573883, 0.06951090693473816, 0.06456652283668518, 0.05965253338217735, 0.05486428737640381, 0.05769607052206993, 0.05246838554739952, 0.04762233421206474, 0.04343637079000473, 0.04164456948637962, 0.037851959466934204, 0.03430584445595741, 0.03165426850318909, 0.03010466881096363, 0.028140543028712273, 0.02688485011458397, 0.02598506212234497, 0.020877771079540253, 0.02026207186281681, 0.019998610019683838, 0.019917715340852737 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.987575202150765e-06, "first_to_last_ratio": 4.909869812054598, "imbalance_abs_log_ratio": 1.591247426598407, "last_quartile_mean": 1.6268405289565635e-06, "mean": 4.1047768952751085e-06, "normalized": [ 3.0399271535189927, 2.272270352549729, 2.3432241010161556, 1.9571821071865307, 1.8062987058242357, 1.478417794211781, 1.3560422313507472, 1.3140126510883237, 1.3159728793374756, 1.1652738240702745, 1.0556436223006813, 1.0336446190031157, 1.0145704481914697, 0.9357289247102711, 0.8598075807486498, 0.8414029385582174, 0.8707583924282057, 0.7424610436170234, 0.6796317067393598, 0.6258542173489432, 0.5891788165353382, 0.546613535826505, 0.5056719741938386, 0.47978148465269643, 0.4608872008433993, 0.4431059730541786, 0.4372858876540878, 0.43183612852165015, 0.35323279260322304, 0.34612823922654523, 0.3468995237902729, 0.35125314929808227 ], "peak_layer": 1, "peak_normalized": 3.0399271535189927, "population_cv": 0.659872101277405, "rest_mean": 4.472362248903934e-06, "spike_contrast": 0.4739812805188056, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.119815985679452e-06, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.2478222743084189e-05, 9.327162842964754e-06, 9.618412150302902e-06, 8.033795893425122e-06, 7.414453193632653e-06, 6.068575203244109e-06, 5.56625082026585e-06, 5.393728770286543e-06, 5.401775069913128e-06, 4.7831890697125345e-06, 4.33318155046436e-06, 4.242880550009431e-06, 4.164585334365256e-06, 3.840958470391342e-06, 3.529318291839445e-06, 3.453771341810352e-06, 3.5742689306061948e-06, 3.047636937481002e-06, 2.7897365271201124e-06, 2.568991931184428e-06, 2.4184475932997884e-06, 2.24372661250527e-06, 2.0756706362590194e-06, 1.9693959529831773e-06, 1.891839133350004e-06, 1.818851160351187e-06, 1.7949610082723666e-06, 1.772590962900722e-06, 1.4499418057312141e-06, 1.4207791991793783e-06, 1.42394515023625e-06, 1.4418158116313862e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001181762008855003, "first_to_last_ratio": 4.901880355055182, "imbalance_abs_log_ratio": 1.589618877434572, "last_quartile_mean": 2.4108340539896744e-05, "mean": 6.302894070131515e-05, "normalized": [ 2.3744696571174315, 2.136630154802725, 2.0361131254128098, 1.9194114069249477, 1.8802010360140171, 1.7011133103712184, 1.531914150175555, 1.4197583943464076, 1.391271208233108, 1.2656343297221408, 1.162210793277347, 1.0817658397942223, 1.0342567205447348, 0.961623134639753, 0.8926032889087109, 0.8268877508247753, 0.8608377110603633, 0.7877810435620806, 0.7225748772463415, 0.6647855101906275, 0.649524097104473, 0.5935729705103481, 0.541611739989691, 0.5034768505696168, 0.479620762906317, 0.4516427176844916, 0.43439036804465014, 0.4220573087232059, 0.32687806257013674, 0.3181526251894958, 0.31448965055456213, 0.31273940298369435 ], "peak_layer": 1, "peak_normalized": 2.3744696571174315, "population_cv": 0.5918793634752633, "rest_mean": 6.823978502426244e-05, "spike_contrast": 0.5112909037593624, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.489038135739975e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001496603072155267, 0.00013466953532770276, 0.00012833405344281346, 0.0001209784677485004, 0.0001185070796054788, 0.00010721936996560544, 9.655492613092065e-05, 8.948586764745414e-05, 8.769035048317164e-05, 7.977159111760557e-05, 7.325291517190635e-05, 6.818255496909842e-05, 6.518810550915077e-05, 6.061008753022179e-05, 5.625983976642601e-05, 5.211785901337862e-05, 5.425768904387951e-05, 4.9653004680294544e-05, 4.5543129090219736e-05, 4.19007265008986e-05, 4.0938815800473094e-05, 3.7412275560200214e-05, 3.4137214242946357e-05, 3.173361255903728e-05, 3.0229988624341786e-05, 2.846656207111664e-05, 2.7379164748708718e-05, 2.6601825084071606e-05, 2.0602778022293933e-05, 2.0052822947036475e-05, 1.982194953598082e-05, 1.9711633285623975e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.71794661381864e-05, "first_to_last_ratio": 4.097119254175958, "imbalance_abs_log_ratio": 1.410284105823798, "last_quartile_mean": 2.3718974262010306e-05, "mean": 5.355534244699811e-05, "normalized": [ 2.371829633380241, 2.0724601623849908, 1.9622627494202263, 1.8551438939252523, 1.7690721277799024, 1.623771959149998, 1.4821770640361394, 1.379774949857855, 1.3569338467664578, 1.240304612630695, 1.1353847655737628, 1.0568513894692897, 1.0172105466389505, 0.9484210902295721, 0.8824818807944864, 0.8225686147527733, 0.84894808898371, 0.7863829975557285, 0.7313823810333281, 0.6780532337320588, 0.6794732285544001, 0.6281359052189119, 0.5798405933708942, 0.5480368937974985, 0.5477339289626515, 0.5240982224731324, 0.5122994743538888, 0.5044283664293616, 0.36991721031610375, 0.3633394769331656, 0.3616997646762252, 0.3595809468183485 ], "peak_layer": 1, "peak_normalized": 2.371829633380241, "population_cv": 0.5506248473620988, "rest_mean": 5.755568741329221e-05, "spike_contrast": 0.5551750151042494, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.195347962900996e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012702414824161679, 0.0001109913137042895, 0.00010508965351618826, 9.935286652762443e-05, 9.474326361669227e-05, 8.696166332811117e-05, 7.937850023154169e-05, 7.389431993942708e-05, 7.26710568415001e-05, 6.64249382680282e-05, 6.080591992940754e-05, 5.660003807861358e-05, 5.447705916594714e-05, 5.079301627120003e-05, 4.7261619329219684e-05, 4.4052943849237636e-05, 4.546570562524721e-05, 4.2115010728593916e-05, 3.916943387594074e-05, 3.631337312981486e-05, 3.638942143879831e-05, 3.3640033507253975e-05, 3.105356154264882e-05, 2.9350303520914167e-05, 2.9334078135434538e-05, 2.806825978041161e-05, 2.7436373784439638e-05, 2.7014833904104307e-05, 1.981104287551716e-05, 1.9458770111668855e-05, 1.937095476023387e-05, 1.925748074427247e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002535142939450452, "first_to_last_ratio": 5.8113823462408005, "imbalance_abs_log_ratio": 1.759818467906725, "last_quartile_mean": 4.3623750570986886e-05, "mean": 0.0001317092663839503, "normalized": [ 2.263052967073855, 2.19252412234975, 2.11375794838107, 1.992924119774411, 1.9927385047584378, 1.7937674940199644, 1.5956161724502433, 1.4540367722249958, 1.4568245329887053, 1.3084040040884366, 1.1868508068097534, 1.107565794677377, 1.0648325776249805, 0.9875608255051405, 0.9068516600656852, 0.8192947368434954, 0.8729470886658927, 0.7913471921200238, 0.7163422081401906, 0.6492365241541177, 0.6063969651654865, 0.5480260697153099, 0.48690171913384017, 0.4424994861082037, 0.407070855223836, 0.38050431936474255, 0.3622742455318211, 0.34702058581961753, 0.2955446253020686, 0.28773683461603417, 0.28448542301034185, 0.28506281829217234 ], "peak_layer": 1, "peak_normalized": 2.263052967073855, "population_cv": 0.6276890850082345, "rest_mean": 0.0001439489836532933, "spike_contrast": 0.45581977353542075, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.561479312949814e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00029806504608131945, 0.0002887757436838001, 0.00027840150869451463, 0.00026248657377436757, 0.0002624621265567839, 0.00023625580070074648, 0.0002101574355037883, 0.00019151011656504124, 0.0001918772904900834, 0.0001723289315123111, 0.00015631924907211214, 0.00014587667828891426, 0.000140248317620717, 0.0001300709118368104, 0.00011944076686631888, 0.0001079087087418884, 0.00011497522064018995, 0.00010422775812912732, 9.434890671400353e-05, 8.551046630600467e-05, 7.98680994194001e-05, 7.218011160148308e-05, 6.412946822820231e-05, 5.828128269058652e-05, 5.361500370781869e-05, 5.011594475945458e-05, 4.771487510879524e-05, 4.570582677843049e-05, 3.8925965782254934e-05, 3.78976073989179e-05, 3.7469366361619905e-05, 3.754541467060335e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ace6376bf838ff2894b72bcf39de9a912ced4cfce16da0bea85a55c0a2d375f6", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.01706349104642868, 0.012710319831967354, 0.013922877609729767, 0.011416541412472725, 0.008325628004968166, 0.00801781378686428, 0.007913725450634956, 0.008828459307551384, 0.008378393016755581, 0.008136541582643986, 0.008199729025363922, 0.009040302596986294, 0.00936646107584238, 0.009016151539981365, 0.009497544728219509, 0.009566655382514, 0.011108046397566795, 0.0114391278475523, 0.012216469272971153, 0.01392423827201128, 0.015176027081906796, 0.017532862722873688, 0.018514130264520645, 0.022848375141620636, 0.0230264812707901, 0.02426965720951557, 0.025484448298811913, 0.027965376153588295, 0.02452327311038971, 0.02779858186841011, 0.03124845214188099, 0.03395458683371544 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.540868192430935e-05, "first_to_last_ratio": 5.391981827595804, "imbalance_abs_log_ratio": 1.6849130033513229, "last_quartile_mean": 6.566914180439198e-06, "mean": 1.820804214958116e-05, "normalized": [ 2.169983600580502, 2.229739182249934, 2.1980957529981846, 2.095076135992008, 2.0397373476740466, 1.7793673196646478, 1.594082179228277, 1.4512998336206846, 1.4154497014096232, 1.2782832528040422, 1.1723161764872307, 1.078459317768477, 1.0399977909539722, 0.9562526437490421, 0.8686051898876579, 0.800284903251136, 0.826346801214095, 0.7461827311573033, 0.6773489568290562, 0.6148904219045332, 0.5952269402216567, 0.5351818973485719, 0.49310547712451824, 0.45940569949796806, 0.4422680977368002, 0.4169925152644191, 0.40092337249837284, 0.3905686784090242, 0.31321857584806834, 0.3079679144493542, 0.3062939345842967, 0.307047657592496 ], "peak_layer": 2, "peak_normalized": 2.229739182249934, "population_cv": 0.6294018473328937, "rest_mean": 1.9876985822554195e-05, "spike_contrast": 0.46263283566326563, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.195746315526776e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.951115286326967e-05, 4.059918501297943e-05, 4.002302011940628e-05, 3.814723459072411e-05, 3.713962360052392e-05, 3.239879515604116e-05, 2.902511550928466e-05, 2.6425328542245552e-05, 2.577256782387849e-05, 2.327503534615971e-05, 2.1345582354115322e-05, 1.9636632714536972e-05, 1.893632361316122e-05, 1.7411488443030976e-05, 1.5815599908819422e-05, 1.4571621250070166e-05, 1.5046157386677805e-05, 1.3586526620201766e-05, 1.2333198355918285e-05, 1.1195950719411485e-05, 1.083791721612215e-05, 9.744614544615615e-06, 8.978485311672557e-06, 8.36487834021682e-06, 8.052836165006738e-06, 7.592617293994408e-06, 7.300029665202601e-06, 7.111490958777722e-06, 5.703097031073412e-06, 5.607492767012445e-06, 5.577012871071929e-06, 5.590736691374332e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013440313705359586, "first_to_last_ratio": 5.096756498301237, "imbalance_abs_log_ratio": 1.6286043566835047, "last_quartile_mean": 2.6370327304903185e-05, "mean": 7.053613683183357e-05, "normalized": [ 2.118631459598408, 2.1956122951818604, 2.1120000110372854, 2.0289234994136547, 1.9870377238438375, 1.7654736087105818, 1.588703129633362, 1.4472245249652265, 1.4174081755766987, 1.2876606578775502, 1.181827430204035, 1.0872648180837396, 1.0499502322069625, 0.9708212520819881, 0.8822750951131371, 0.8137021952597919, 0.8431394603174627, 0.7640069214416272, 0.6933471494528902, 0.6299834356426122, 0.6115185731312205, 0.5509443526589552, 0.5080623331524264, 0.4736371435182702, 0.4570686826070521, 0.4315058915031859, 0.41528384146383185, 0.4052875681754842, 0.32485087604930224, 0.31941934544156514, 0.3181727511181896, 0.31925556553780515 ], "peak_layer": 2, "peak_normalized": 2.1956122951818604, "population_cv": 0.6066332113433263, "rest_mean": 7.68027995137446e-05, "spike_contrast": 0.47779714518019456, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6696158349514005e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001494400785304606, 0.00015487000928260386, 0.00014897232176735997, 0.00014311242557596415, 0.00014015796477906406, 0.0001245296880370006, 0.00011206098133698106, 0.00010208162711933255, 9.99784970190376e-05, 9.082660835701972e-05, 8.336154132848606e-05, 7.66914599807933e-05, 7.405943324556574e-05, 6.847798067610711e-05, 6.223227683221921e-05, 5.739540938520804e-05, 5.9471800341270864e-05, 5.389009675127454e-05, 4.890602940577082e-05, 4.4436597818275914e-05, 4.3134157749591395e-05, 3.886148624587804e-05, 3.5836754250340164e-05, 3.34085343638435e-05, 3.223985913791694e-05, 3.0436758606811054e-05, 2.9292517865542322e-05, 2.8587419365067035e-05, 2.291372584295459e-05, 2.2530606656800956e-05, 2.244267670903355e-05, 2.2519054255099036e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11919004376977682, "first_to_last_ratio": 5.096756387349086, "imbalance_abs_log_ratio": 1.628604334914336, "last_quartile_mean": 0.0233854700345546, "mean": 0.06255215115379542, "normalized": [ 2.118631411517831, 2.195612379432328, 2.1119998486113145, 2.028923550569329, 1.987037719744253, 1.7654735296881534, 1.5887031196057697, 1.4472244970610906, 1.4174081917206085, 1.2876605843188982, 1.1818275057210115, 1.0872648442748674, 1.049950213270653, 0.9708212769375442, 0.8822751544186832, 0.8137022140448229, 0.843139511676983, 0.7640069424918614, 0.6933471370791401, 0.6299834087164704, 0.6115185748160756, 0.5509443408044579, 0.5080623343135898, 0.47363716064572287, 0.4570686754958606, 0.4315059038674738, 0.415283849867723, 0.4052875513382056, 0.3248508776130179, 0.31941937674712706, 0.31817274273479984, 0.31925557085433326 ], "peak_layer": 2, "peak_normalized": 2.195612379432328, "population_cv": 0.6066332011935822, "rest_mean": 0.0681094902533072, "spike_contrast": 0.47779714538168394, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.032542520016431806, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.13252495229244232, 0.13734027743339539, 0.132110133767128, 0.12691353261470795, 0.1242934837937355, 0.1104341670870781, 0.09937679767608643, 0.0905270054936409, 0.08866193145513535, 0.08054593950510025, 0.07392585277557373, 0.06801075488328934, 0.06567664444446564, 0.06072695925831795, 0.05518820881843567, 0.050898823887109756, 0.0527401901781559, 0.0477902777493, 0.04337035492062569, 0.03940681740641594, 0.03825180232524872, 0.03446275368332863, 0.031780391931533813, 0.02962702326476574, 0.028590628877282143, 0.026991622522473335, 0.02597689814865589, 0.02535160817205906, 0.020320121198892593, 0.01998036913573742, 0.019902389496564865, 0.0199701227247715 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.877708355863433e-06, "first_to_last_ratio": 4.956106826106702, "imbalance_abs_log_ratio": 1.6006205184390925, "last_quartile_mean": 1.5894952696271503e-06, "mean": 3.999532470544409e-06, "normalized": [ 2.814716234830519, 2.2772104806158655, 2.4900910014868787, 2.034929380837634, 1.946817467216516, 1.4831174218104588, 1.3741082206996111, 1.3362682498464256, 1.335883261215519, 1.175420204513858, 1.0516337389632453, 1.0321730525814614, 1.0275683111275076, 0.9193973073548298, 0.8640024348029502, 0.8181451837262547, 0.853266641386687, 0.7182658185841684, 0.6535137098090048, 0.6032328415810824, 0.5571593897123995, 0.5134489675885875, 0.48188120740570606, 0.4583873216846789, 0.454112395874878, 0.4406721596566101, 0.43551085607971224, 0.4317150626061903, 0.3525239188448322, 0.350593887609604, 0.35373076042061513, 0.3605031095257097 ], "peak_layer": 1, "peak_normalized": 2.814716234830519, "population_cv": 0.663747605253736, "rest_mean": 4.375045717905677e-06, "spike_contrast": 0.45068350411145924, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.971760934793565e-06, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1257548976573162e-05, 9.107777259487193e-06, 9.959199815057218e-06, 8.138766133924946e-06, 7.78635967435548e-06, 5.931776286161039e-06, 5.495790446730098e-06, 5.344448254618328e-06, 5.3429084800882265e-06, 4.701131274487125e-06, 4.206043286103522e-06, 4.128209639020497e-06, 4.109792826056946e-06, 3.67715938409674e-06, 3.455605792623828e-06, 3.2721982279326767e-06, 3.4126676382584265e-06, 2.8727274639095413e-06, 2.613749302327051e-06, 2.4126493372023106e-06, 2.2283770704234485e-06, 2.0535558178380597e-06, 1.9272995359642664e-06, 1.8333349771637586e-06, 1.8162372725782916e-06, 1.7624826114115422e-06, 1.741839810165402e-06, 1.7266584109165706e-06, 1.4099308600634686e-06, 1.4022116374690086e-06, 1.4147576621326152e-06, 1.4418438922803034e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00011129347331007011, "first_to_last_ratio": 4.7212247859996666, "imbalance_abs_log_ratio": 1.5520682544953823, "last_quartile_mean": 2.3573008775201743e-05, "mean": 5.946740844819942e-05, "normalized": [ 2.156243192585657, 2.162278083675196, 2.065029760843484, 1.9785360583158715, 1.9104190175998155, 1.7137610234833742, 1.5574858033444396, 1.4282764408526964, 1.3933238942763717, 1.272869675359645, 1.1719605800561352, 1.0826910772254577, 1.0407375480008605, 0.9652826895223989, 0.8847702267230649, 0.8215667819990273, 0.8441113654099096, 0.7717750152456593, 0.7061028737915589, 0.6469451209977515, 0.637162587501422, 0.5782978066221776, 0.536179040993063, 0.5029771071348699, 0.48641749518891064, 0.46322823897876675, 0.44748104445301945, 0.4384066844549108, 0.33861230782778606, 0.3333797707047228, 0.3318178248031526, 0.33187386202882385 ], "peak_layer": 2, "peak_normalized": 2.162278083675196, "population_cv": 0.5844863507438243, "rest_mean": 6.444277332101828e-05, "spike_contrast": 0.5058819857516098, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.260043813497759e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001282261946471408, 0.00012858507398050278, 0.00012280196824576706, 0.00011765841190936044, 0.0001136076680268161, 0.0001019129267660901, 9.261964441975579e-05, 8.493589848512784e-05, 8.285736112156883e-05, 7.5694260885939e-05, 6.96934584993869e-05, 6.438483251258731e-05, 6.188996485434473e-05, 5.7402859965804964e-05, 5.261499245534651e-05, 4.885644739260897e-05, 5.019711534259841e-05, 4.5895460061728954e-05, 4.1990108002210036e-05, 3.8472149753943086e-05, 3.7890407838858664e-05, 3.4389871871098876e-05, 3.188517803209834e-05, 2.9910745070083067e-05, 2.8925987862749025e-05, 2.7546982892090455e-05, 2.661053804331459e-05, 2.6070909370901063e-05, 2.013639641518239e-05, 1.9825230992864817e-05, 1.9732346117962152e-05, 1.9735678506549448e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.178715026791906e-05, "first_to_last_ratio": 3.9336058830578526, "imbalance_abs_log_ratio": 1.3695565327057526, "last_quartile_mean": 2.3334099296334898e-05, "mean": 5.082815437162935e-05, "normalized": [ 2.1389396648642625, 2.0813842206789825, 1.9784383506454575, 1.9084846997435907, 1.8069300868376652, 1.6378043812798742, 1.5010746826985348, 1.3936066186205618, 1.3599265725007244, 1.2387510684837115, 1.136508771737783, 1.057443309955879, 1.023880001173994, 0.9508578975343295, 0.8771227728274181, 0.8254997469475742, 0.8391602342847156, 0.7782386607285642, 0.7170818947183901, 0.6619181702664086, 0.6688760305343471, 0.6150432978276499, 0.5789519206743307, 0.5514510803162503, 0.5572369864630182, 0.5414044044824042, 0.5311915694026624, 0.5242909684053321, 0.38287903611819507, 0.38027305935168987, 0.3789296852891365, 0.37642015460656175 ], "peak_layer": 1, "peak_normalized": 2.1389396648642625, "population_cv": 0.5381630565291086, "rest_mean": 5.464674312086707e-05, "spike_contrast": 0.5527827167839162, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0207775125745684e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010871835547732189, 0.00010579291847534478, 0.00010056036990135908, 9.700475493445992e-05, 9.184292139252648e-05, 8.324657392222434e-05, 7.629685569554567e-05, 7.083445234457031e-05, 6.912255776114762e-05, 6.296343053691089e-05, 5.776664329459891e-05, 5.374789179768413e-05, 5.204193075769581e-05, 4.8330352001357824e-05, 4.458253170014359e-05, 4.195862857159227e-05, 4.265296593075618e-05, 3.955643478548154e-05, 3.64479492418468e-05, 3.364407893968746e-05, 3.399773413548246e-05, 3.12615156872198e-05, 2.9427057597786188e-05, 2.8029240638716146e-05, 2.8323327569523826e-05, 2.75185866485117e-05, 2.699948709050659e-05, 2.6648742277757265e-05, 1.946103475347627e-05, 1.9328577764099464e-05, 1.926029653986916e-05, 1.913274172693491e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00023838880770199466, "first_to_last_ratio": 5.615601076105167, "imbalance_abs_log_ratio": 1.7255486307434573, "last_quartile_mean": 4.245116497259005e-05, "mean": 0.0001231883321679561, "normalized": [ 2.1010282721933486, 2.235453689609075, 2.1351018794085928, 2.0660271287710446, 2.05547953479137, 1.8078923375666518, 1.6237057602904945, 1.456570280035237, 1.4451526778837593, 1.3168179083075675, 1.1987172226380065, 1.1000538071149966, 1.0715582930268015, 0.9881627826227964, 0.8804515183677988, 0.8100379119774714, 0.8454810290012423, 0.7652053218316441, 0.6893306865502836, 0.6202261087486645, 0.5903640380365041, 0.5227167386885779, 0.4731335967935112, 0.4445012321327385, 0.4104702393707161, 0.39051495112024115, 0.3718063497323823, 0.3604546331284706, 0.3081351294824555, 0.30324722511926294, 0.3046138998706107, 0.307587815787682 ], "peak_layer": 2, "peak_normalized": 2.235453689609075, "population_cv": 0.6295499532649526, "rest_mean": 0.0001348630006153447, "spike_contrast": 0.4459720032746656, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.014512255205773e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00025882216868922114, 0.00027538181166164577, 0.0002630196395330131, 0.00025451043620705605, 0.00025321109569631517, 0.00022271124180406332, 0.00020002160454168916, 0.0001794324634829536, 0.0001780259481165558, 0.00016221660189330578, 0.00014766797539778054, 0.00013551379379350692, 0.00013200347893871367, 0.00012173012510174885, 0.0001084613541024737, 9.978721936931834e-05, 0.00010415339784231037, 9.426436736248434e-05, 8.491749758832157e-05, 7.640461990376934e-05, 7.272596121765673e-05, 6.439260323531926e-05, 5.828453868161887e-05, 5.4757365433033556e-05, 5.056514419266023e-05, 4.810688551515341e-05, 4.5802204112987965e-05, 4.440380507730879e-05, 3.79586526833009e-05, 3.735651989700273e-05, 3.752487828023732e-05, 3.789123002206907e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7b7c9c05d1cbde4434ba686237dd1c945f87f81120c1f3f234a3adcdf1d68140", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.01049753837287426, 0.028129518032073975, 0.040982723236083984, 0.05135732889175415, 0.014544809237122536, 0.023477500304579735, 0.029818039387464523, 0.03540094941854477, 0.01795734278857708, 0.024303091689944267, 0.031844362616539, 0.03825921565294266, 0.018456056714057922, 0.02560909278690815, 0.0305727981030941, 0.037264734506607056, 0.017418840900063515, 0.02998829260468483, 0.03708711639046669, 0.04352690279483795, 0.01969025284051895, 0.030230700969696045, 0.041474223136901855, 0.056570712476968765, 0.012774706818163395, 0.03524070605635643, 0.05920406058430672, 0.08430205285549164, 0.005331806372851133, 0.032364487648010254, 0.059356335550546646, 0.09272631257772446 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.746988363673154e-05, "first_to_last_ratio": 5.5431540155630445, "imbalance_abs_log_ratio": 1.7125636556484518, "last_quartile_mean": 6.7596685084936325e-06, "mean": 1.9353528273313714e-05, "normalized": [ 2.342683119997902, 2.192172693070629, 2.1504636799030705, 2.019405278342902, 1.9995383922272116, 1.773068785590967, 1.5680654500633198, 1.443203173299378, 1.419437780389259, 1.2773938398900322, 1.1654352925993345, 1.0793229270979923, 1.0342651399222493, 0.9566605213538533, 0.8835696516415633, 0.8115626455796351, 0.8495787965648868, 0.7730098730619744, 0.7009135788914922, 0.6382359583361713, 0.6098093032158669, 0.5543264714540777, 0.5014428165322925, 0.4622493313451456, 0.4400298915440255, 0.4102977502891804, 0.39090442010501514, 0.37729033323761296, 0.3032525161403676, 0.2939411057398244, 0.29000420505871766, 0.2884652775140499 ], "peak_layer": 1, "peak_normalized": 2.342683119997902, "population_cv": 0.6284613723388149, "rest_mean": 2.1096881331080414e-05, "spike_contrast": 0.4711322780552658, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.939421761373524e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.5339183998294175e-05, 4.242627619532868e-05, 4.161905962973833e-05, 3.9082617149688303e-05, 3.869812280754559e-05, 3.431513687246479e-05, 3.034759902220685e-05, 2.7931073418585584e-05, 2.747112921497319e-05, 2.472207779646851e-05, 2.2555284886038862e-05, 2.088870678562671e-05, 2.001667962758802e-05, 1.8514756447984837e-05, 1.7100190234486945e-05, 1.5706600606790744e-05, 1.6442347259726375e-05, 1.4960468433855567e-05, 1.3565150766225997e-05, 1.2352117664704565e-05, 1.1801961591118015e-05, 1.072817303793272e-05, 9.704687727207784e-06, 8.946155503508635e-06, 8.516130947100464e-06, 7.940709110698663e-06, 7.5653797466657124e-06, 7.301899131562095e-06, 5.869006145076128e-06, 5.688797500624787e-06, 5.61260458198376e-06, 5.582820904237451e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00014312549137684982, "first_to_last_ratio": 5.283834952428555, "imbalance_abs_log_ratio": 1.6646521508256433, "last_quartile_mean": 2.7087426587968366e-05, "mean": 7.517451274452469e-05, "normalized": [ 2.316758705258724, 2.1576744140515562, 2.080949754300773, 1.9631709090738392, 1.9581116297712768, 1.756527727676563, 1.5601436404205942, 1.4379416037311736, 1.4182272361289956, 1.2807847399809855, 1.1719416125963091, 1.0858232362724045, 1.0426842467677646, 0.968528909567598, 0.8948112457108309, 0.8229911987427623, 0.8654616765749427, 0.7870364110125266, 0.7143514220202891, 0.6515701232514104, 0.6246851525269175, 0.5677858273151118, 0.5145937075954704, 0.474826748264951, 0.45157784417649677, 0.42211454965836737, 0.4032790856857949, 0.3897818669587336, 0.3131725537818108, 0.3039381495313863, 0.2999843051410979, 0.2987697664525418 ], "peak_layer": 1, "peak_normalized": 2.316758705258724, "population_cv": 0.6100374517355251, "rest_mean": 8.176350510674874e-05, "spike_contrast": 0.4842497143049554, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.959395398851484e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017416120681446046, 0.00016220212273765355, 0.00015643438382539898, 0.00014758041652385145, 0.00014720008766744286, 0.00013204611605033278, 0.0001172830379800871, 0.00010809655941557139, 0.00010661454143701121, 9.628236875869334e-05, 8.810013969196007e-05, 8.162623271346092e-05, 7.838328019715846e-05, 7.280868885573e-05, 6.726699939463288e-05, 6.186796235851943e-05, 6.506065983558074e-05, 5.9165078710066155e-05, 5.3701020078733563e-05, 4.898146653431468e-05, 4.6960401959950104e-05, 4.268302291166037e-05, 3.86843312298879e-05, 3.569486943888478e-05, 3.394714440219104e-05, 3.173225559294224e-05, 3.0316308766487055e-05, 2.9301661925273947e-05, 2.354259413550608e-05, 2.2848402295494452e-05, 2.255117396998685e-05, 2.2459871615865268e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12692511826753616, "first_to_last_ratio": 5.2838350418377695, "imbalance_abs_log_ratio": 1.6646521677469164, "last_quartile_mean": 0.02402140060439706, "mean": 0.06666550948284566, "normalized": [ 2.3167586877655544, 2.1576745567833733, 2.080949753728949, 1.963171017440623, 1.9581116128663651, 1.7565277011358755, 1.560143677115271, 1.4379415661053954, 1.418227211166364, 1.280784747512944, 1.1719416103835674, 1.0858232130311325, 1.0426842731319477, 0.9685288473950006, 0.8948111881196698, 0.8229911687817946, 0.8654616682400994, 0.7870363710328231, 0.7143513801530333, 0.6515701150530128, 0.6246851968826057, 0.5677858388521877, 0.5145936933053726, 0.47482679570383546, 0.4515778146632158, 0.4221145224430977, 0.40327910912708953, 0.3897818593837784, 0.31317256095995943, 0.3039381697062612, 0.29998430147060634, 0.29876977055919474 ], "peak_layer": 1, "peak_normalized": 2.3167586877655544, "population_cv": 0.6100374622681133, "rest_mean": 0.0725086936382232, "spike_contrast": 0.4842497262327929, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03511231504380703, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.15444789826869965, 0.14384247362613678, 0.1387275755405426, 0.13087579607963562, 0.13053850829601288, 0.1170998141169548, 0.1040077731013298, 0.09586110711097717, 0.094546839594841, 0.08538416773080826, 0.07812808454036713, 0.0723869577050209, 0.06951107829809189, 0.06456746906042099, 0.05965304374694824, 0.05486512556672096, 0.05769644305109978, 0.052468180656433105, 0.047622598707675934, 0.043437253683805466, 0.04164495691657066, 0.037851732224226, 0.03430565074086189, 0.03165457025170326, 0.030104665085673332, 0.0281404796987772, 0.026884807273745537, 0.025985006242990494, 0.020877808332443237, 0.02026219293475151, 0.01999860629439354, 0.01991763897240162 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.987624144334404e-06, "first_to_last_ratio": 4.909876092861521, "imbalance_abs_log_ratio": 1.591248705818253, "last_quartile_mean": 1.6268484159809304e-06, "mean": 4.104807253213494e-06, "normalized": [ 3.039754669455638, 2.272302514080732, 2.3431733144286278, 1.9573395693607292, 1.8063799565900491, 1.4783923475555005, 1.35608582196543, 1.313927156720024, 1.3158812773467614, 1.1653219275062023, 1.0555854083106069, 1.0336141529504548, 1.0145705888314611, 0.9357245523705084, 0.8598480281460422, 0.8414154936952141, 0.8707475211933563, 0.7424554418110071, 0.6796411377088122, 0.625913455746514, 0.5892207668944752, 0.5466000211947281, 0.5056892833715051, 0.4797947755131972, 0.4608548499137519, 0.44312659764186396, 0.43727941318247243, 0.43184315462220935, 0.3532260811866724, 0.34613656390182396, 0.34690003248276174, 0.3512541243208678 ], "peak_layer": 1, "peak_normalized": 3.039754669455638, "population_cv": 0.6598595805076404, "rest_mean": 4.472391268521918e-06, "spike_contrast": 0.4739866087898418, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1198535705480025e-06, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.2477607015171088e-05, 9.327363841293845e-06, 9.618274816602934e-06, 8.034501661313698e-06, 7.41484154787031e-06, 6.068515631341143e-06, 5.566470917983679e-06, 5.393417723098537e-06, 5.401439011620823e-06, 4.783421900356188e-06, 4.332974640419707e-06, 4.242786872055149e-06, 4.164616711932467e-06, 3.840968929580413e-06, 3.5295104225951945e-06, 3.4538484214863274e-06, 3.5742507407121593e-06, 3.047636482733651e-06, 2.789795871649403e-06, 2.569254093032214e-06, 2.418637677692459e-06, 2.2436877316067694e-06, 2.075757038255688e-06, 1.969465074580512e-06, 1.8917203306045849e-06, 1.8189492720921407e-06, 1.7949477069123532e-06, 1.7726329133438412e-06, 1.4499249800792313e-06, 1.420823878106603e-06, 1.423957769475237e-06, 1.4418304772334523e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00011817605991382152, "first_to_last_ratio": 4.9018741377821335, "imbalance_abs_log_ratio": 1.5896176090892555, "last_quartile_mean": 2.4108342358886148e-05, "mean": 6.302900811760992e-05, "normalized": [ 2.374456497049209, 2.136637566259215, 2.0361178738653583, 1.9193980409609805, 1.880195792666385, 1.7010985617651888, 1.5319147049553272, 1.4197582610211454, 1.3912798786840082, 1.265638517025675, 1.16221093542691, 1.081772301653134, 1.0342582693755369, 0.961625165193809, 0.8926056818824999, 0.8268911375942922, 0.8608363285490097, 0.787779046563255, 0.722577567522019, 0.6647906287621113, 0.6495296937526279, 0.5935714121143937, 0.5416093713850725, 0.5034789094073873, 0.4796213754232066, 0.4516410802217136, 0.4343902208726426, 0.4220558183434384, 0.3268775109220198, 0.31815320839698785, 0.3144898336461675, 0.3127388087392738 ], "peak_layer": 1, "peak_normalized": 2.374456497049209, "population_cv": 0.5918778380779056, "rest_mean": 6.823984788045184e-05, "spike_contrast": 0.5112917815905388, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.489047339826357e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001496596378274262, 0.00013467014650814235, 0.0001283344900002703, 0.00012097775470465422, 0.0001185068758786656, 0.00010721855505835265, 9.655506437411532e-05, 8.948595495894551e-05, 8.769099076744169e-05, 7.977194036357105e-05, 7.325300248339772e-05, 6.818303518230096e-05, 6.51882728561759e-05, 6.0610280343098566e-05, 5.626005076919682e-05, 5.2118128223810345e-05, 5.425765994004905e-05, 4.96529319207184e-05, 4.554334736894816e-05, 4.1901093936758116e-05, 4.093921234016307e-05, 3.74122173525393e-05, 3.413710146560334e-05, 3.17337762680836e-05, 3.0230059564928524e-05, 2.84664893115405e-05, 2.7379184757592157e-05, 2.660175960045308e-05, 2.0602765289368108e-05, 2.005288115469739e-05, 1.9821982277790084e-05, 1.9711616914719343e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.71798262980883e-05, "first_to_last_ratio": 4.097156236797104, "imbalance_abs_log_ratio": 1.4102931322767618, "last_quartile_mean": 2.3718848069620435e-05, "mean": 5.355637233606103e-05, "normalized": [ 2.3716408308104846, 2.0723215416211813, 1.962393612474412, 1.8549484527834383, 1.7689695012129536, 1.624026031735852, 1.4819536083276605, 1.3800136078097518, 1.3569801642067971, 1.240510630022215, 1.1354021945249317, 1.0569709979561241, 1.016947599552204, 0.9485224054224415, 0.8823796609912891, 0.8224276054007224, 0.8490865037613989, 0.7863957259021926, 0.731366006996846, 0.6781886854392298, 0.6795365133883927, 0.6281649905506911, 0.579800641551788, 0.5480420803864836, 0.5476558755782504, 0.5241602836307675, 0.5123889675770241, 0.504358312217994, 0.3698923675924519, 0.36330593005557427, 0.36164828236666086, 0.35960038815179535 ], "peak_layer": 1, "peak_normalized": 2.3716408308104846, "population_cv": 0.5506073147861105, "rest_mean": 5.755683479138598e-05, "spike_contrast": 0.5551708184288228, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.1953875077306296e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012701647938229144, 0.00011098602408310398, 0.00010509868297958747, 9.934431000147015e-05, 9.473958925809711e-05, 8.697694283910096e-05, 7.936805923236534e-05, 7.390852260868996e-05, 7.267493492690846e-05, 6.64372491883114e-05, 6.0808022681158036e-05, 5.660753231495619e-05, 5.446402428788133e-05, 5.079941911390051e-05, 4.7257053665816784e-05, 4.4046239054296166e-05, 4.5473992940969765e-05, 4.211650229990482e-05, 3.91693101846613e-05, 3.6321325751487166e-05, 3.639351052697748e-05, 3.364223812241107e-05, 3.1052019039634615e-05, 2.935114571300801e-05, 2.933046198450029e-05, 2.807212331390474e-05, 2.744169432844501e-05, 2.7011601559934206e-05, 1.981009336304851e-05, 1.9457347661955282e-05, 1.9368570065125823e-05, 1.9258892280049622e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002534907544031739, "first_to_last_ratio": 5.810954685354278, "imbalance_abs_log_ratio": 1.7597448749820743, "last_quartile_mean": 4.362291019788245e-05, "mean": 0.00013170268027806742, "normalized": [ 2.263139397683698, 2.19239908258678, 2.1138453104279673, 1.9929654416215998, 1.9922306787749795, 1.793581963413166, 1.5951409704801716, 1.4544554310298363, 1.4566936402003607, 1.307837648386261, 1.187117991027653, 1.1077423893871614, 1.0650861466665695, 0.9874986152737254, 0.9069117045975197, 0.8193320614345609, 0.8732068621805286, 0.7912116928785183, 0.7164957030827894, 0.6492442961101281, 0.6064825348153222, 0.5480344706704015, 0.48698153412089457, 0.44258326813255405, 0.4070688650357849, 0.380575498977065, 0.3622952070517462, 0.3470762520438106, 0.29549026518772864, 0.28774337874464384, 0.28440730681701687, 0.28512439115905686 ], "peak_layer": 1, "peak_normalized": 2.263139397683698, "population_cv": 0.6276378428727017, "rest_mean": 0.0001439405388265624, "spike_contrast": 0.4558704910453311, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.561824411619455e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002980615245178342, 0.00028874483541585505, 0.0002783990930765867, 0.000262478890363127, 0.00026238212012685835, 0.00023621955187991261, 0.0002100843412335962, 0.00019155567861162126, 0.0001918504567584023, 0.0001722457236610353, 0.0001563466212246567, 0.00014589264173991978, 0.00014027470024302602, 0.00013005621440242976, 0.00011944270227104425, 0.00010790822852868587, 0.00011500368418637663, 0.00010420470061944798, 9.436440450372174e-05, 8.550721395295113e-05, 7.987537537701428e-05, 7.21776086720638e-05, 6.413677328964695e-05, 5.828940265928395e-05, 5.361206058296375e-05, 5.012281326344237e-05, 4.771524982061237e-05, 4.571087265503593e-05, 3.8916859921300784e-05, 3.7896574212936684e-05, 3.74572045984678e-05, 3.75516465282999e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4c56d08d88ae21cddacd7e29fc552875e42d5e83a4e1d9907d475a90abe4ea95", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.021243074908852577, 0.03624089062213898, 0.04951142147183418, 0.05851924046874046, 0.017167888581752777, 0.02674153819680214, 0.03352004662156105, 0.03946679085493088, 0.020600929856300354, 0.027865095064044, 0.035815127193927765, 0.04293550178408623, 0.021564343944191933, 0.029616132378578186, 0.035480253398418427, 0.04259813204407692, 0.021436138078570366, 0.03586751967668533, 0.04331304505467415, 0.051551222801208496, 0.025608548894524574, 0.04091007262468338, 0.05466190353035927, 0.07478146255016327, 0.02793189324438572, 0.05498877540230751, 0.08125203847885132, 0.10916247963905334, 0.026290444657206535, 0.05761713907122612, 0.08842677623033524, 0.12436132878065109 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.540896136655647e-05, "first_to_last_ratio": 5.3920073915355005, "imbalance_abs_log_ratio": 1.6849177444428016, "last_quartile_mean": 6.566934871443664e-06, "mean": 1.8208185622370365e-05, "normalized": [ 2.1699627058464013, 2.229719814658003, 2.1981171939571054, 2.095121565395603, 2.0397310655629486, 1.7793850670583011, 1.594066721449286, 1.451277409043192, 1.4154824041645198, 1.278296457066803, 1.1723320139174174, 1.078457313436146, 1.0400037819469097, 0.956235318728193, 0.8685919520760419, 0.8002742017652755, 0.826347582617928, 0.746171556877762, 0.6773531600124774, 0.6148977645726124, 0.5952208514919352, 0.5351825254697238, 0.4931013918631679, 0.45940307857109275, 0.44226511234925703, 0.416991702053712, 0.40092308550387123, 0.3905686478293239, 0.31321395997822676, 0.3079637645255534, 0.30629421840547977, 0.3070466118057301 ], "peak_layer": 2, "peak_normalized": 2.229719814658003, "population_cv": 0.6294044766409223, "rest_mean": 1.9877152664304487e-05, "spike_contrast": 0.4626298218476693, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.19576359592611e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.951108374167234e-05, 4.059915227117017e-05, 4.002372588729486e-05, 3.814836236415431e-05, 3.7139801861485466e-05, 3.239937359467149e-05, 2.9025062758591957e-05, 2.6425128453411162e-05, 2.5773366360226646e-05, 2.3275459170690738e-05, 2.1346038920455612e-05, 1.9636750948848203e-05, 1.8936581909656525e-05, 1.7411310182069428e-05, 1.5815483493497595e-05, 1.457154121453641e-05, 1.5046290172904264e-05, 1.3586430213763379e-05, 1.2333372069406323e-05, 1.1196172636118717e-05, 1.08378917502705e-05, 9.744702765601687e-06, 8.97848167369375e-06, 8.364896530110855e-06, 8.052845259953756e-06, 7.592662313982146e-06, 7.300081961147953e-06, 7.11154643795453e-06, 5.703057922801236e-06, 5.607461389445234e-06, 5.577061983785825e-06, 5.5907617024786305e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013440396742225857, "first_to_last_ratio": 5.096772825661455, "imbalance_abs_log_ratio": 1.6286075601589056, "last_quartile_mean": 2.6370405748821213e-05, "mean": 7.053641201082428e-05, "normalized": [ 2.1186363977530145, 2.195594858531836, 2.112032207136982, 2.0289327073081966, 1.9870506023056362, 1.7654824002594274, 1.5886840377574483, 1.4472277500593902, 1.4174334883240383, 1.287660792002894, 1.1818213754945048, 1.0872652182409996, 1.0499307664886113, 0.9708194245660434, 0.8822672176250175, 0.8136967514782698, 0.8431298271983457, 0.7640029609304118, 0.6933490863748281, 0.6299871670381194, 0.6115155169679979, 0.5509413780821966, 0.5080592679889337, 0.47363704932987183, 0.4570686014801849, 0.43150580695093105, 0.4152831239229507, 0.4052856775987878, 0.32485051130850606, 0.319418692435047, 0.31817385655546626, 0.3192554805051111 ], "peak_layer": 2, "peak_normalized": 2.195594858531836, "population_cv": 0.606636113722199, "rest_mean": 7.680312336122841e-05, "spike_contrast": 0.47779529155407807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.669617071864195e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001494410098530352, 0.00015486938355024904, 0.00014897517394274473, 0.0001431136333849281, 0.0001401594199705869, 0.00012453079398255795, 0.00011206007184227929, 0.00010208225285168737, 9.998067253036425e-05, 9.082697215490043e-05, 8.336143946507946e-05, 7.669178739888594e-05, 7.405834912788123e-05, 6.847811891930178e-05, 6.22319639660418e-05, 5.739524931414053e-05, 5.947135286987759e-05, 5.3890027629677206e-05, 4.890635682386346e-05, 4.4437034375732765e-05, 4.31341104558669e-05, 3.886142803821713e-05, 3.5836677852785215e-05, 3.340865805512294e-05, 3.223997919121757e-05, 3.0436871384154074e-05, 2.9292581530171447e-05, 2.8587397537194192e-05, 2.2913789507583715e-05, 2.2530648493557237e-05, 2.244284223706927e-05, 2.2519136109622195e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11919078417122364, "first_to_last_ratio": 5.096772875478614, "imbalance_abs_log_ratio": 1.6286075699331606, "last_quartile_mean": 0.023385539650917053, "mean": 0.06255239609163254, "normalized": [ 2.118636455801519, 2.195594967924862, 2.112032314032563, 2.0289327576260647, 1.9870505449879214, 1.7654823390288878, 1.5886840348658555, 1.447227644229518, 1.4174334908870143, 1.2876607830122906, 1.18182132958956, 1.0872652321152292, 1.0499307368395738, 0.9708194407763235, 0.8822672330699993, 0.8136967647333175, 0.8431298378216533, 0.7640029384223331, 0.6933490673920089, 0.629987135566792, 0.6115155251770166, 0.5509413496931483, 0.5080592728974551, 0.4736370628781186, 0.4570685830511319, 0.43150579241003445, 0.41528311705209375, 0.40528569634831185, 0.3248505286851896, 0.31941869175924303, 0.31817384926992365, 0.31925548205504595 ], "peak_layer": 2, "peak_normalized": 2.195594967924862, "population_cv": 0.6066361252561874, "rest_mean": 0.06810977848039733, "spike_contrast": 0.4777952875249589, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0325425311923027, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.13252578675746918, 0.1373397260904312, 0.13211268186569214, 0.1269146054983139, 0.12429477274417877, 0.11043515056371689, 0.09937599301338196, 0.09052755683660507, 0.08866386115550995, 0.08054626733064651, 0.07392575591802597, 0.06801104545593262, 0.06567568331956863, 0.0607270821928978, 0.055187929421663284, 0.050898682326078415, 0.052739791572093964, 0.04779021441936493, 0.04337064549326897, 0.03940720483660698, 0.038251761347055435, 0.03446270152926445, 0.03178032487630844, 0.029627133160829544, 0.02859073504805565, 0.026991721242666245, 0.02597695402801037, 0.025351591408252716, 0.02032017894089222, 0.019980404525995255, 0.019902536645531654, 0.01997019536793232 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.877724669924646e-06, "first_to_last_ratio": 4.955998917619468, "imbalance_abs_log_ratio": 1.6005987453688633, "last_quartile_mean": 1.5895331699766757e-06, "mean": 3.999530946430241e-06, "normalized": [ 2.814718217044885, 2.277240228243144, 2.490109915020848, 2.0349008216507296, 1.9468032006740146, 1.4832109937263445, 1.3741164759470665, 1.3361972416551804, 1.3359113994263188, 1.175339925313911, 1.0516196997910439, 1.032190500940567, 1.0275201527334392, 0.9193727005250851, 0.8639824117206115, 0.8181664731809545, 0.8532322879917306, 0.7182487530200276, 0.6535170287501896, 0.6032571758931335, 0.5571458443102854, 0.5134825911003963, 0.4819078263273953, 0.4583689632360092, 0.45412774789772825, 0.4406778136182034, 0.43551940742882095, 0.4317317420710209, 0.35252700938669523, 0.3506082337328722, 0.35372873491469664, 0.3605184827266523 ], "peak_layer": 1, "peak_normalized": 2.814718217044885, "population_cv": 0.6637468990729557, "rest_mean": 4.37503757877022e-06, "spike_contrast": 0.45069215893423337, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9717951317943514e-06, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1257552614551969e-05, 9.107892765314318e-06, 9.959271665138658e-06, 8.138648809108417e-06, 7.786299647705164e-06, 5.932148269494064e-06, 5.495821369549958e-06, 5.3441622185346205e-06, 5.343018983694492e-06, 4.700808403867995e-06, 4.20598553318996e-06, 4.12827785112313e-06, 4.1095986489381175e-06, 3.6770595670532202e-06, 3.4555243928480195e-06, 3.2722821288189152e-06, 3.412528940316406e-06, 2.872658114938531e-06, 2.6137615805055248e-06, 2.4127457436406985e-06, 2.228322045993991e-06, 2.0536895135592204e-06, 1.9274052647233475e-06, 1.833260853345564e-06, 1.816297981349635e-06, 1.7625045529712224e-06, 1.7418733477825299e-06, 1.7267244629692868e-06, 1.4099426834945916e-06, 1.4022684808878694e-06, 1.4147490219329484e-06, 1.4419048284253222e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00011129383165098261, "first_to_last_ratio": 4.721235661143692, "imbalance_abs_log_ratio": 1.5520705579509457, "last_quartile_mean": 2.357303037570091e-05, "mean": 5.946752537511202e-05, "normalized": [ 2.1562533904228016, 2.1622669804365335, 2.0650538414267587, 1.978536205665008, 1.9104188094756904, 1.7137689101965365, 1.5574747881012931, 1.4282752231039315, 1.3933371827625038, 1.272867417306948, 1.1719582757105038, 1.0826854002026853, 1.0407336663933602, 0.9652771209998985, 0.884767569421755, 0.821562719573055, 0.844106891600729, 0.7717762506724927, 0.7061036265861128, 0.6469448277682184, 0.6371632923208572, 0.5782976483691459, 0.5361789043793784, 0.502977158155649, 0.48641626348807704, 0.4632276952202235, 0.44748102106422594, 0.4384054248026578, 0.33861167262559666, 0.3333792069668021, 0.33181827353856846, 0.33187434124200266 ], "peak_layer": 2, "peak_normalized": 2.1622669804365335, "population_cv": 0.5844879671939917, "rest_mean": 6.444290172819617e-05, "spike_contrast": 0.5058818301813628, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.2600493068457584e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012822705321013927, 0.0001285846665268764, 0.00012280364171601832, 0.0001176586520159617, 0.00011360787902958691, 0.0001019135961541906, 9.261917148251086e-05, 8.493599307257682e-05, 8.285831427201629e-05, 7.569427543785423e-05, 6.96934584993869e-05, 6.43846215098165e-05, 6.188985571498051e-05, 5.740264168707654e-05, 5.2614937885664403e-05, 4.885630187345669e-05, 5.019694799557328e-05, 4.5895623770775273e-05, 4.1990235331468284e-05, 3.8472207961604e-05, 3.789052425418049e-05, 3.438993007875979e-05, 3.1885232601780444e-05, 2.9910806915722787e-05, 2.8925971491844393e-05, 2.7547004719963297e-05, 2.661058897501789e-05, 2.6070885724038817e-05, 2.0136398234171793e-05, 1.9825236449833028e-05, 1.973241160158068e-05, 1.973574580915738e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.179371863865526e-05, "first_to_last_ratio": 3.933988193388572, "imbalance_abs_log_ratio": 1.3696537187901656, "last_quartile_mean": 2.3333501303568482e-05, "mean": 5.082899730268764e-05, "normalized": [ 2.1394359800622813, 2.0814587808021265, 1.9787622602269084, 1.9084551972599093, 1.8068306956003535, 1.6376936233321315, 1.5012763891845262, 1.393543999273884, 1.35980639449679, 1.238717928620473, 1.1365151179184183, 1.05713604656592, 1.023790231850539, 0.9508624555295617, 0.8771210384666132, 0.8254319480230166, 0.8391731577705832, 0.7782341287088573, 0.7170084501774274, 0.6619086246953101, 0.6688703776588982, 0.6150814814369804, 0.5789313330893375, 0.5514835190919826, 0.5572498614580683, 0.5413707333622386, 0.5311388145382818, 0.5242249796321137, 0.38285690475582773, 0.3802104967302831, 0.3789175322683099, 0.37650151741204796 ], "peak_layer": 1, "peak_normalized": 2.1394359800622813, "population_cv": 0.5382143248139301, "rest_mean": 5.464754158984525e-05, "spike_contrast": 0.552794458326558, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0208858152036556e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010874538565985858, 0.00010579846275504678, 0.00010057850158773363, 9.700486407382414e-05, 9.18393925530836e-05, 8.324232476297766e-05, 7.630837353644893e-05, 7.08324441802688e-05, 6.911759555805475e-05, 6.296279025264084e-05, 5.7767923863139004e-05, 5.3733165259473026e-05, 5.2038230933249e-05, 4.833138518733904e-05, 4.458318289835006e-05, 4.195587825961411e-05, 4.265433017280884e-05, 3.955686042900197e-05, 3.64448205800727e-05, 3.36441516992636e-05, 3.39980106218718e-05, 3.126397496089339e-05, 2.94264991680393e-05, 2.8031354304403067e-05, 2.8324451704975218e-05, 2.7517331545823254e-05, 2.6997253371519037e-05, 2.6645830075722188e-05, 1.9460232579149306e-05, 1.932571831275709e-05, 1.9259998225606978e-05, 1.913719461299479e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002383605042268755, "first_to_last_ratio": 5.6146902736182, "imbalance_abs_log_ratio": 1.7253864261379666, "last_quartile_mean": 4.245301033733995e-05, "mean": 0.0001231808448665106, "normalized": [ 2.100827328644271, 2.2345440763144846, 2.136083407505072, 2.0655627440949633, 2.055234003018172, 1.8075987971575753, 1.623843438405382, 1.4566679111096852, 1.4453319547522105, 1.317016319283833, 1.1984433593272132, 1.1000521536406067, 1.071769085471528, 0.9883476552878133, 0.8803897945877148, 0.8099451507686356, 0.8457679211262041, 0.7651810707046709, 0.6892705769434541, 0.6200948164612059, 0.5902902932604684, 0.5228157886141485, 0.47313831489002084, 0.4446663790889937, 0.4105354433359864, 0.3905597157431388, 0.3717521617725102, 0.3606343113512262, 0.3081187728953204, 0.3032816350953125, 0.304634216769742, 0.3076014025784363 ], "peak_layer": 2, "peak_normalized": 2.2345440763144846, "population_cv": 0.6294889602855173, "rest_mean": 0.000134853615842682, "spike_contrast": 0.4460234990313686, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.014788159518503e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00025878168526105583, 0.0002752530272118747, 0.00026312455884180963, 0.0002544377639424056, 0.000253165460890159, 0.00022266154701355845, 0.00020002640667371452, 0.0001794335839804262, 0.00017803721129894257, 0.0001622311829123646, 0.00014762526552658528, 0.00013550535368267447, 0.00013202142145019025, 0.00012174549920018762, 0.00010844715870916843, 9.976972796721384e-05, 0.00010418240708531812, 9.425565076526254e-05, 8.490493200952187e-05, 7.638380338903517e-05, 7.271245704032481e-05, 6.440089055104181e-05, 5.82815773668699e-05, 5.4774380259914324e-05, 5.0570102757774293e-05, 4.810947575606406e-05, 4.579274536808953e-05, 4.442323916009627e-05, 3.795433076447807e-05, 3.7358488043537363e-05, 3.752510019694455e-05, 3.789060065173544e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "787f9e4002b423fdbb7f8faf6fa82a8ff8c9854d40d44697d8afa4d95a3e169c", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04377923160791397, 0.023813940584659576, 0.029579024761915207, 0.034570932388305664, 0.037983279675245285, 0.02627667225897312, 0.02788778953254223, 0.03023744560778141, 0.031768180429935455, 0.02424551732838154, 0.025113960728049278, 0.026638666167855263, 0.027698444202542305, 0.022435806691646576, 0.022681118920445442, 0.023235918954014778, 0.023750189691781998, 0.019677624106407166, 0.020757248625159264, 0.02137788012623787, 0.02207154594361782, 0.019271831959486008, 0.020465917885303497, 0.021911408752202988, 0.024193286895751953, 0.02259150519967079, 0.024306390434503555, 0.02611992135643959, 0.02851196378469467, 0.027104467153549194, 0.03068944811820984, 0.03286270797252655 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.8306669605626666e-05, "first_to_last_ratio": 5.668716287294882, "imbalance_abs_log_ratio": 1.7349626877491615, "last_quartile_mean": 4.993488502691434e-06, "mean": 1.5805165769933183e-05, "normalized": [ 0.8783128680331268, 2.1826435477445063, 2.2956954594453394, 2.767580901011819, 2.1125287729616464, 1.9101651305644987, 1.2546030463288333, 0.9262767734665253, 0.8744163240087035, 1.1038744529856963, 1.057809216514194, 0.8349821291845769, 1.5598261518374243, 1.5571623184207217, 1.076399772260337, 1.4673081766428846, 1.1246096789003457, 0.7371277404470634, 0.6369864324222737, 0.6349327395421065, 0.6011227365145811, 0.8094112388497428, 0.6205992994135143, 0.4481028505293834, 0.5856560220554665, 0.7623288521260666, 0.3625047598024719, 0.13435636426712916, 0.1475298295833587, 0.2669943453689258, 0.11457117055614831, 0.1535808982105884 ], "peak_layer": 4, "peak_normalized": 2.767580901011819, "population_cv": 0.670823858490635, "rest_mean": 1.6937932451178818e-05, "spike_contrast": 0.5719839608011006, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.688225691206754e-06, "top_five_layers": [ 4, 3, 2, 5, 6 ], "top_five_spike_overlap": 0 }, "values": [ 1.3881880477129016e-05, 3.449704308877699e-05, 3.628384729381651e-05, 4.374207492219284e-05, 3.338886745041236e-05, 3.0190476536517963e-05, 1.9829209122690372e-05, 1.4639957953477278e-05, 1.3820294952893164e-05, 1.7446918718633242e-05, 1.6718850019969977e-05, 1.3197030966694001e-05, 2.465331090206746e-05, 2.4611208573332988e-05, 1.701267683529295e-05, 2.3191048967419192e-05, 1.7774642401491292e-05, 1.1650426131382119e-05, 1.0067676157632377e-05, 1.0035217201220803e-05, 9.50084449868882e-06, 1.2792878806067165e-05, 9.808674803934991e-06, 7.082339834596496e-06, 9.256390512746293e-06, 1.2048733879055362e-05, 5.729447821067879e-06, 2.1235246094875038e-06, 2.3317334125749767e-06, 4.219889888190664e-06, 1.8108163430952118e-06, 2.4273715553135844e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.993926990617183e-05, "first_to_last_ratio": 3.4893308817982382, "imbalance_abs_log_ratio": 1.2497099934197489, "last_quartile_mean": 2.004374829311928e-05, "mean": 4.464254881497709e-05, "normalized": [ 0.8066620510828731, 2.249059806280304, 1.8293400677886618, 1.6505903941799722, 1.482941789801698, 1.867443438314272, 1.4782170879001166, 1.1689490929675366, 1.2145199991696674, 1.4010197387507275, 1.1581363399144262, 1.0008260463099323, 1.1290384086543988, 1.2173363380864748, 0.9961077007288228, 1.0849452654707106, 1.086573542898278, 1.0033179684088052, 0.8130676737841358, 0.6422200474001272, 0.6427230931215812, 1.0345398071106366, 0.8415727559232208, 0.6089870444535808, 0.8527565388110598, 1.0992667074243434, 0.5132675404314736, 0.1873435614965349, 0.20684501755119733, 0.3674535488046901, 0.15217388752900993, 0.21275769945073061 ], "peak_layer": 2, "peak_normalized": 2.249059806280304, "population_cv": 0.5008866669948363, "rest_mean": 4.632808737302117e-05, "spike_contrast": 0.7671510441468363, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5540640601539054e-05, "top_five_layers": [ 2, 6, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.601144999265671e-05, 0.0001004037621896714, 8.166640327544883e-05, 7.368656224571168e-05, 6.62023012409918e-05, 8.336743485415354e-05, 6.599137850571424e-05, 5.218486694502644e-05, 5.4219268349697813e-05, 6.25450920779258e-05, 5.170215808902867e-05, 4.467942562769167e-05, 5.040315227233805e-05, 5.43449968972709e-05, 4.446878665476106e-05, 4.843472197535448e-05, 4.850741242989898e-05, 4.479067138163373e-05, 3.629741331678815e-05, 2.8670339816017076e-05, 2.8692797059193254e-05, 4.6184493839973584e-05, 3.7569952837657183e-05, 2.71867338597076e-05, 3.806922541116364e-05, 4.907406764687039e-05, 2.291357122885529e-05, 8.36349408928072e-06, 9.234088793164119e-06, 1.6404062989749946e-05, 6.793430202378659e-06, 9.498045983491465e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06202284339815378, "first_to_last_ratio": 3.489330820237693, "imbalance_abs_log_ratio": 1.2497099757772414, "last_quartile_mean": 0.017774996580556035, "mean": 0.039589458669070154, "normalized": [ 0.8066620927081755, 2.249059736772125, 1.8293399648688806, 1.6505903951156085, 1.4829417588396991, 1.867443458890533, 1.4782170964630472, 1.1689491284714175, 1.2145200544167176, 1.4010197265931286, 1.1581363231796216, 1.0008260498990915, 1.1290383875745214, 1.2173363145296596, 0.996107692090837, 1.0849452712811893, 1.086573543692374, 1.0033179540826904, 0.8130677223960504, 0.6422200351976349, 0.6427230832969295, 1.0345398705514788, 0.8415727462776835, 0.6089870555080975, 0.8527565801245708, 1.0992667100332167, 0.513267522372738, 0.18734356688581966, 0.20684500840965173, 0.36745355899142085, 0.15217388494831066, 0.2127577055370788 ], "peak_layer": 2, "peak_normalized": 2.249059736772125, "population_cv": 0.5008866561011148, "rest_mean": 0.04108421097475069, "spike_contrast": 0.7671510653512432, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03151779621839523, "top_five_layers": [ 2, 6, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.03193531557917595, 0.08903905749320984, 0.0724225789308548, 0.06534598022699356, 0.05870886147022247, 0.07393107563257217, 0.0585218146443367, 0.04627806320786476, 0.04808219149708748, 0.055465612560510635, 0.0458499900996685, 0.03962216153740883, 0.044698018580675125, 0.048193685710430145, 0.03943536430597305, 0.04295239597558975, 0.04301685839891434, 0.0397208146750927, 0.032188910990953445, 0.025425143539905548, 0.025445058941841125, 0.04095687344670296, 0.033317409455776215, 0.02410946786403656, 0.03376017138361931, 0.04351937398314476, 0.02031998336315155, 0.007416830398142338, 0.008188881911337376, 0.014547287486493587, 0.006024481728672981, 0.00842296238988638 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.5216306564980187e-05, "first_to_last_ratio": 7.030788105499711, "imbalance_abs_log_ratio": 1.9502988055846384, "last_quartile_mean": 2.1642391061504895e-06, "mean": 1.0416225535436752e-05, "normalized": [ 1.2614020219005322, 1.2435865799883328, 1.4443414880126273, 2.0469556474720734, 1.6464946614743052, 1.773065362472431, 1.0834782895079982, 1.1872943029799679, 1.2246868580585597, 1.5602141574184534, 1.327224315217328, 1.3409942697906458, 1.3449072126725554, 2.065589232705634, 1.2862289608083182, 1.79783161732383, 1.738192374488952, 1.4057167409862177, 0.9202432421079245, 0.7366699550385442, 0.583019146679299, 0.5088209255652885, 0.40525183944794096, 0.40558477226377354, 0.26054664683684153, 0.30292479006597633, 0.2336502506334751, 0.2297086901983557, 0.15260828683083258, 0.17440390432050698, 0.1566888857901092, 0.15167457094237005 ], "peak_layer": 14, "peak_normalized": 2.065589232705634, "population_cv": 0.6121354643087803, "rest_mean": 1.1510614630996887e-05, "spike_contrast": 0.3915103201593089, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.506524419412017e-06, "top_five_layers": [ 14, 4, 16, 6, 17 ], "top_five_spike_overlap": 0 }, "values": [ 1.3139047950971872e-05, 1.295347829000093e-05, 1.5044586689327843e-05, 2.1321551685105078e-05, 1.7150259736808948e-05, 1.8468648704583757e-05, 1.1285754226264544e-05, 1.236712523677852e-05, 1.2756614523823373e-05, 1.625154254725203e-05, 1.3824667803419288e-05, 1.3968098755867686e-05, 1.4008856851432938e-05, 2.151564331143163e-05, 1.339765094598988e-05, 1.8726619600784034e-05, 1.8105403796653263e-05, 1.4642262613051571e-05, 9.585461157257669e-06, 7.673320396861527e-06, 6.072858923289459e-06, 5.29999351783772e-06, 4.221194558340358e-06, 4.224662461638218e-06, 2.7139126359543297e-06, 3.1553329336020397e-06, 2.4337537070096005e-06, 2.3926975245558424e-06, 1.5896023342065746e-06, 1.816630401663133e-06, 1.6321067732860683e-06, 1.579876538926328e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.190560773371544e-05, "first_to_last_ratio": 5.742880674184589, "imbalance_abs_log_ratio": 1.7479609440560961, "last_quartile_mean": 9.038252869686403e-06, "mean": 3.1702642814934734e-05, "normalized": [ 0.7426774192575819, 2.3203889953928316, 1.8256902534826824, 1.543739301356747, 1.5734163075504266, 2.109348732800137, 1.64875428919227, 1.334099715975053, 1.3912665329141354, 1.6086824817138736, 1.3339909299684964, 1.1314499757741263, 1.1960452245153466, 1.3418021783506682, 1.0883087504651683, 1.0855161172841996, 1.0978647060664168, 1.0745881727648192, 0.87642608484069, 0.6880064264214529, 0.6659679650868783, 0.8134892489335985, 0.743952040109719, 0.4837711990268687, 0.5137493175424936, 0.6236389627256068, 0.32417337747920116, 0.18870731593182713, 0.15601040790107645, 0.21053976663531476, 0.12020463563707022, 0.14373316690322271 ], "peak_layer": 2, "peak_normalized": 2.3203889953928316, "population_cv": 0.5767311578617529, "rest_mean": 3.3791577186127176e-05, "spike_contrast": 0.6043635400030921, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0422397210495546e-05, "top_five_layers": [ 2, 6, 3, 7, 10 ], "top_five_spike_overlap": 0 }, "values": [ 2.354483694944065e-05, 7.356246351264417e-05, 5.787920599686913e-05, 4.8940615670289844e-05, 4.9881455197464675e-05, 6.687192944809794e-05, 5.226986831985414e-05, 4.229448677506298e-05, 4.410682595334947e-05, 5.0999486120417714e-05, 4.2291037971153855e-05, 3.586995444493368e-05, 3.791779454331845e-05, 4.253867518855259e-05, 3.450226358836517e-05, 3.4413729736115783e-05, 3.480521263554692e-05, 3.406728501431644e-05, 2.778502312139608e-05, 2.1811621991219e-05, 2.111294452333823e-05, 2.57897590927314e-05, 2.3585245799040422e-05, 1.533682552690152e-05, 1.628721111046616e-05, 1.977100328076631e-05, 1.0277152796334121e-05, 5.982520633551758e-06, 4.945942237100098e-06, 6.674667019979097e-06, 3.810804628301412e-06, 4.556721250992268e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.938292866223492e-05, "first_to_last_ratio": 9.18982862399783, "imbalance_abs_log_ratio": 2.2180972880977254, "last_quartile_mean": 4.285491087330229e-06, "mean": 2.3341486304673253e-05, "normalized": [ 0.6673051270990256, 2.308149633585561, 1.872866846541638, 1.4969405906506341, 1.5875800328734282, 2.3234736824333693, 1.784836015034525, 1.4568500921073086, 1.548779422738, 1.7929545502301463, 1.477805011955662, 1.2083348392465652, 1.2158304842751781, 1.3732002989348482, 1.1418738689499803, 1.0221659591685974, 1.0518199886832604, 1.1236379858609276, 0.9217957770992289, 0.7176364137990805, 0.6755225547783117, 0.7056040273764278, 0.6474467916351409, 0.4087919922057317, 0.30494607832868875, 0.3212400761898423, 0.2149387803961774, 0.18637979615916483, 0.12482940461335829, 0.12661742786534272, 0.09393484526198889, 0.09591160392285902 ], "peak_layer": 6, "peak_normalized": 2.3234736824333693, "population_cv": 0.6451544009698722, "rest_mean": 2.5293256915877766e-05, "spike_contrast": 0.5061398398295045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.280192500416888e-05, "top_five_layers": [ 6, 2, 3, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 1.557589348522015e-05, 5.3875643061473966e-05, 4.371549584902823e-05, 3.4940818295581266e-05, 3.705647759488784e-05, 5.423332913778722e-05, 4.166072540101595e-05, 3.4005046472884715e-05, 3.615081368479878e-05, 4.185022407909855e-05, 3.449416544754058e-05, 2.820433110173326e-05, 2.8379290597513318e-05, 3.205253597116098e-05, 2.665303327376023e-05, 2.3858872737037018e-05, 2.45510418608319e-05, 2.622738065838348e-05, 2.151608350686729e-05, 1.6750700524426065e-05, 1.576770046085585e-05, 1.646984674152918e-05, 1.5112370419956278e-05, 9.541812687530182e-06, 7.117894710972905e-06, 7.498220838897396e-06, 5.016990598960547e-06, 4.350381459516939e-06, 2.9137038382032188e-06, 2.9554389584518503e-06, 2.192578904214315e-06, 2.2387193894246593e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012773637081409106, "first_to_last_ratio": 4.324151431064901, "imbalance_abs_log_ratio": 1.4642159202240765, "last_quartile_mean": 2.9540216814893938e-05, "mean": 8.050168943896097e-05, "normalized": [ 0.8871717025827831, 2.337156289954447, 1.7098779659370984, 1.4615981277611254, 1.6595169230948639, 1.8749716736387922, 1.5249505044111165, 1.238788125932271, 1.2542052402145494, 1.4655117877457478, 1.219570330002609, 1.060650662474342, 1.227816664029005, 1.4059417450611726, 1.0036779465715522, 1.2092205201021657, 1.1438081416422972, 1.014111450824222, 0.8576088870341769, 0.6691271722099967, 0.6720599542707117, 0.8395852269377942, 0.8228857633365203, 0.5045750826570258, 0.6467370801446735, 0.9037936178533763, 0.4164763377402587, 0.19712216248220002, 0.18271060059268188, 0.2652336386100389, 0.14113561188950058, 0.1824030622608841 ], "peak_layer": 2, "peak_normalized": 2.337156289954447, "population_cv": 0.5285114363123314, "rest_mean": 8.501621491800675e-05, "spike_contrast": 0.6601476189717621, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.6123251852113754e-05, "top_five_layers": [ 2, 6, 3, 5, 7 ], "top_five_spike_overlap": 0 }, "values": [ 7.141882088035345e-05, 0.0001881450298242271, 0.00013764806499239057, 0.00011766111856559291, 0.0001335939159616828, 0.00015093838737811893, 0.00012276109191589057, 9.972453699447215e-05, 0.0001009656407404691, 0.00011797617480624467, 9.817747195484117e-05, 8.538417023373768e-05, 9.884131577564403e-05, 0.00011318068573018536, 8.079777035163715e-05, 9.734429477248341e-05, 9.207848779624328e-05, 8.163768507074565e-05, 6.903896428411826e-05, 5.386586781241931e-05, 5.410196172306314e-05, 6.758802919648588e-05, 6.624369416385889e-05, 4.0619146602693945e-05, 5.2063427574466914e-05, 7.275691314134747e-05, 3.352704879944213e-05, 1.586866710567847e-05, 1.4708512026118115e-05, 2.1351756004150957e-05, 1.1361655197106302e-05, 1.4683754670841154e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ad0dc0ee191d8145bd7a67f8b576436d75a4c4558d60d541ecf9bca80ffb9f93", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.017109708860516548, 0.02606838569045067, 0.030872320756316185, 0.034764163196086884, 0.025188947096467018, 0.026965361088514328, 0.028485212475061417, 0.030349206179380417, 0.022602196782827377, 0.023387571796774864, 0.02450813725590706, 0.025169003754854202, 0.020798001438379288, 0.02118327096104622, 0.021654823794960976, 0.022377775982022285, 0.018887808546423912, 0.01950881816446781, 0.020426612347364426, 0.021234560757875443, 0.01860968954861164, 0.019469313323497772, 0.020844420418143272, 0.02213398739695549, 0.0209572184830904, 0.022487536072731018, 0.023487187922000885, 0.024889562278985977, 0.023824406787753105, 0.025868752971291542, 0.0271493811160326, 0.028410611674189568 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.406384535584948e-05, "first_to_last_ratio": 1.6917204830931702, "imbalance_abs_log_ratio": 0.5257460484196427, "last_quartile_mean": 8.313338696552819e-06, "mean": 1.2088094919704417e-05, "normalized": [ 2.384784726621517, 1.1311337715172713, 0.9796665485569107, 0.8224622106532729, 1.211988479625169, 1.0625055316005352, 0.8419214674481715, 0.8731050481608821, 1.162924153782958, 1.064627644161191, 0.9632127839920408, 0.9176822777828343, 1.137313817736726, 1.047865701151065, 1.0364330031252973, 0.9266530092349528, 1.3076030994322017, 1.245640723171692, 1.1356311002249702, 1.055064557020718, 1.1753193075391253, 1.1259884855902011, 0.9820417647039537, 0.9065953024025623, 0.9143977245534269, 0.8160033291510744, 0.782411126229384, 0.7182286763887548, 0.6000700810180117, 0.5741865902243426, 0.5489069666673982, 0.5476309905313882 ], "peak_layer": 1, "peak_normalized": 2.384784726621517, "population_cv": 0.31980213589111106, "rest_mean": 1.2041379991219233e-05, "spike_contrast": 1.0248290098413306, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2340355533524417e-05, "top_five_layers": [ 1, 17, 18, 5, 21 ], "top_five_spike_overlap": 1 }, "values": [ 2.8827504138462245e-05, 1.3673252396984026e-05, 1.1842302228615154e-05, 9.942001270246692e-06, 1.4650631783297285e-05, 1.284366771869827e-05, 1.017722661345033e-05, 1.055417669704184e-05, 1.4057537555345334e-05, 1.2869320016761776e-05, 1.1643407560768537e-05, 1.1093030479969457e-05, 1.3747957382292952e-05, 1.2666700058616698e-05, 1.2528500519692898e-05, 1.1201469533261843e-05, 1.5806430383236147e-05, 1.5057423297548667e-05, 1.3727616533287801e-05, 1.2753720511682332e-05, 1.4207371350494213e-05, 1.3611055692308582e-05, 1.1871014066855423e-05, 1.0959010069200303e-05, 1.105332648876356e-05, 9.863925697572995e-06, 9.457859960093629e-06, 8.682016414240934e-06, 7.253704097820446e-06, 6.9408220042532776e-06, 6.635239515162539e-06, 6.619815394515172e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.4028067552280845e-05, "first_to_last_ratio": 1.6345037059364802, "imbalance_abs_log_ratio": 0.4913392145007449, "last_quartile_mean": 3.305472318970715e-05, "mean": 4.737582446523447e-05, "normalized": [ 2.3514155771111755, 1.133401461379931, 0.9385744423540482, 0.7603362412028586, 1.162677024813552, 1.060905926051319, 0.8429727875353996, 0.8730309155458181, 1.162986103649938, 1.0649320906402773, 0.9658838036496159, 0.9259372261987153, 1.1426484858464174, 1.062663413586317, 1.038525930654914, 0.9333315438116647, 1.325499141503536, 1.241150951130487, 1.1365919996589382, 1.0669045895397178, 1.1830135675602507, 1.1385949073084929, 0.990415755644998, 0.9159030297690505, 0.924477682983016, 0.8252864902247735, 0.7951806757653125, 0.7255609942258914, 0.6087012425945892, 0.5836005087504837, 0.560707980014089, 0.5581875092944122 ], "peak_layer": 1, "peak_normalized": 2.3514155771111755, "population_cv": 0.3142051905018903, "rest_mean": 4.71084055822791e-05, "spike_contrast": 1.0363306893909856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.881988643319346e-05, "top_five_layers": [ 1, 17, 18, 21, 9 ], "top_five_spike_overlap": 1 }, "values": [ 0.00011140025162603706, 5.3695828682975844e-05, 4.446573802852072e-05, 3.602155629778281e-05, 5.508278263732791e-05, 5.026129292673431e-05, 3.9936530811246485e-05, 4.136055940762162e-05, 5.5097425502026454e-05, 5.045203579356894e-05, 4.57595415355172e-05, 4.386703949421644e-05, 5.413391409092583e-05, 5.034455534769222e-05, 4.920102219330147e-05, 4.4217351387487724e-05, 6.279661465669051e-05, 5.880054959561676e-05, 5.38469830644317e-05, 5.0545484555186704e-05, 5.6046243116725236e-05, 5.394187246565707e-05, 4.692176298703998e-05, 4.339166116551496e-05, 4.379789243103005e-05, 3.9098627894418314e-05, 3.767234011320397e-05, 3.4374050301266834e-05, 2.8837723220931366e-05, 2.7648555260384455e-05, 2.6564002837403677e-05, 2.6444593459018506e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04791263025254011, "first_to_last_ratio": 1.6345036692801245, "imbalance_abs_log_ratio": 0.4913391920741478, "last_quartile_mean": 0.029313259525224566, "mean": 0.04201335500692949, "normalized": [ 2.3514156784172187, 1.133401461661499, 0.9385744066074354, 0.7603362049299565, 1.1626770145536474, 1.0609058838539445, 0.8429727897119484, 0.8730308482354255, 1.1629861153616714, 1.0649320859164637, 0.965883808806204, 0.9259371845017617, 1.142648505828565, 1.062663395935368, 1.0385258681211755, 0.9333315745429377, 1.325499101605811, 1.2411509353845518, 1.1365920452458722, 1.0669045321225719, 1.183013648725627, 1.138594993654493, 0.9904157371924246, 0.9159030239053129, 0.9244776895791931, 0.8252865663984437, 0.7951807151797501, 0.7255609453201705, 0.6087012883299899, 0.5836005027338216, 0.5607079790634799, 0.5581874685732653 ], "peak_layer": 1, "peak_normalized": 2.3514156784172187, "population_cv": 0.31420520653960876, "rest_mean": 0.04177620503361578, "spike_contrast": 1.036330725301315, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.043293964862823484, "top_five_layers": [ 1, 17, 18, 21, 9 ], "top_five_spike_overlap": 1 }, "values": [ 0.09879086166620255, 0.04761799797415733, 0.03943265974521637, 0.03194427490234375, 0.04884796217083931, 0.044572215527296066, 0.03541611507534981, 0.03667895495891571, 0.04886094853281975, 0.044741369783878326, 0.04058001935482025, 0.03890172764658928, 0.04800649732351303, 0.04464605450630188, 0.04363195598125458, 0.039212390780448914, 0.05568866431713104, 0.052144914865493774, 0.04775204509496689, 0.04482423886656761, 0.049702372401952744, 0.047836195677518845, 0.04161068797111511, 0.038480158895254135, 0.038840409368276596, 0.0346730574965477, 0.033408209681510925, 0.030483249574899673, 0.02557358331978321, 0.024519015103578568, 0.02355722337961197, 0.02345132827758789 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.917115407148231e-06, "first_to_last_ratio": 1.7761794884414766, "imbalance_abs_log_ratio": 0.5744647027750681, "last_quartile_mean": 2.2053601185234584e-06, "mean": 3.1665725686025326e-06, "normalized": [ 2.7105980142014188, 1.2584335069420332, 1.1184553125758785, 0.9137724260402323, 1.3069438057161062, 0.9543737632121961, 0.8185107477756304, 0.8150769920988516, 1.106441368261451, 1.0103937914191876, 0.8971971105841021, 0.8698038240167904, 1.023913831911454, 0.9814473768267671, 0.9475786306403914, 0.9043188811303046, 1.2933347278335885, 1.1779841320022393, 1.0971261185238004, 1.0522252088592114, 1.1621449547113154, 1.1289087330174585, 0.9715840445086426, 0.907831047153982, 0.875626869339223, 0.7871309556685123, 0.7978996689976676, 0.7370601330215918, 0.6234295385177253, 0.6039908093128533, 0.5786627593732889, 0.5678009158061047 ], "peak_layer": 1, "peak_normalized": 2.7105980142014188, "population_cv": 0.364237936254411, "rest_mean": 3.161166449461706e-06, "spike_contrast": 1.0109450619113027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.1957656119629975e-06, "top_five_layers": [ 1, 5, 17, 2, 18 ], "top_five_spike_overlap": 0 }, "values": [ 8.583305316278711e-06, 3.9849210224929266e-06, 3.5416699120105477e-06, 2.8935266982443864e-06, 4.13853240388562e-06, 3.0220937787817093e-06, 2.5918736810126575e-06, 2.5810004444792867e-06, 3.5036268855037633e-06, 3.1994852633943083e-06, 2.8410397590050707e-06, 2.754296929197153e-06, 3.2422974527435144e-06, 3.1078243409865536e-06, 3.0005764983798144e-06, 2.8635913622565567e-06, 4.095438271178864e-06, 3.7301722386473557e-06, 3.4741294712148374e-06, 3.331947482365649e-06, 3.680016334328684e-06, 3.5747714264289243e-06, 3.0765913834329695e-06, 2.8747128908435116e-06, 2.7727360247808974e-06, 2.492507292117807e-06, 2.526607204345055e-06, 2.333954398636706e-06, 1.974134875126765e-06, 1.9125807284581242e-06, 1.8323776203033049e-06, 1.7979828044190072e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.53617046787258e-05, "first_to_last_ratio": 1.6235922245104368, "imbalance_abs_log_ratio": 0.48464111692853945, "last_quartile_mean": 2.7939099481955054e-05, "mean": 3.8807642113170004e-05, "normalized": [ 2.437808381833339, 1.182834547067526, 0.9777525684870058, 0.7857025355052735, 1.173309606758207, 1.0576717542347258, 0.859768212992415, 0.8762393870580075, 1.137990945103205, 1.0501479644557203, 0.949971945680228, 0.913039951911735, 1.1178137006265285, 1.0305352456029508, 0.9972430453249006, 0.8949050115271279, 1.2575810394572957, 1.1563573391187367, 1.0783657149583266, 1.0249789523026176, 1.184595619530283, 1.1506932403605004, 1.002430737703081, 0.9427578856689722, 0.9703835498479987, 0.8658087871584248, 0.8379408888026836, 0.7685722951163864, 0.6138918991902961, 0.5823916131103506, 0.561129142648094, 0.5593864908570572 ], "peak_layer": 1, "peak_normalized": 2.437808381833339, "population_cv": 0.3181748559585258, "rest_mean": 3.8447074513425364e-05, "spike_contrast": 1.0600210202615004, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.075470715179108e-05, "top_five_layers": [ 1, 17, 21, 2, 5 ], "top_five_spike_overlap": 1 }, "values": [ 9.460559522267431e-05, 4.590301978169009e-05, 3.7944271753076464e-05, 3.0491262805298902e-05, 4.553337930701673e-05, 4.104574691154994e-05, 3.336557711008936e-05, 3.4004784538410604e-05, 4.416274532559328e-05, 4.075376637047157e-05, 3.686617128551006e-05, 3.543292768881656e-05, 4.337971404311247e-05, 3.999264299636707e-05, 3.8700651202816516e-05, 3.4729153412627056e-05, 4.8803754907567054e-05, 4.487550177145749e-05, 4.1848830733215436e-05, 3.9777016354491934e-05, 4.5971362851560116e-05, 4.4655691453954205e-05, 3.8901973312022164e-05, 3.658621062641032e-05, 3.76582975150086e-05, 3.359999755048193e-05, 3.251851012464613e-05, 2.98264785669744e-05, 2.3823697119951248e-05, 2.2601245291298255e-05, 2.1776098947157152e-05, 2.170847074012272e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.840028375634574e-05, "first_to_last_ratio": 1.5392922378986411, "imbalance_abs_log_ratio": 0.43132272500352103, "last_quartile_mean": 2.4946714347606758e-05, "mean": 3.270594595505827e-05, "normalized": [ 2.461740434904075, 1.197496439702706, 0.9903120585800815, 0.7910638146450573, 1.1699175648793894, 1.0483897843618244, 0.8646493044292616, 0.8692874959222262, 1.129883380630568, 1.0428338098095176, 0.9376272398479191, 0.8900945088315015, 1.0865591482264656, 1.0023234373075094, 0.9670931816451467, 0.8520871001401004, 1.1852808372815498, 1.07429226741598, 1.0193151600779609, 0.9854449496787876, 1.1764973274911055, 1.1637500315753844, 1.0219017710659555, 0.9700968049576442, 1.0255967071543362, 0.9382176643393733, 0.9120563422137004, 0.8495166179296606, 0.6405458356516568, 0.590640394607506, 0.5733315467004426, 0.5721570379956076 ], "peak_layer": 1, "peak_normalized": 2.461740434904075, "population_cv": 0.3136325717265237, "rest_mean": 3.227247995070044e-05, "spike_contrast": 1.0859612410365724, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.504666237859055e-05, "top_five_layers": [ 1, 2, 17, 21, 5 ], "top_five_spike_overlap": 1 }, "values": [ 8.051354961935431e-05, 3.916525383829139e-05, 3.238909266656265e-05, 2.5872490368783474e-05, 3.8263260648818687e-05, 3.428857962717302e-05, 2.8279173420742154e-05, 2.8430869861040264e-05, 3.695390478242189e-05, 3.41068662237376e-05, 3.06659858324565e-05, 2.9111382900737226e-05, 3.553694477886893e-05, 3.278193617006764e-05, 3.1629697332391515e-05, 2.7868314646184444e-05, 3.876573100569658e-05, 3.513574483804405e-05, 3.3337666536681354e-05, 3.222990926587954e-05, 3.847845800919458e-05, 3.806154563790187e-05, 3.3422264095861465e-05, 3.172793367411941e-05, 3.354311047587544e-05, 3.068529622396454e-05, 2.9829665436409414e-05, 2.7784244593931362e-05, 2.0949657482560724e-05, 1.9317452824907377e-05, 1.8751350580714643e-05, 1.871293716249056e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.757476163940737e-05, "first_to_last_ratio": 1.686806792110223, "imbalance_abs_log_ratio": 0.5228372695066706, "last_quartile_mean": 5.7845843457471346e-05, "mean": 8.550922041195008e-05, "normalized": [ 2.337533863407778, 1.1216672060979693, 0.9305333491453122, 0.7843126666651307, 1.1828292881294888, 1.0607795384885752, 0.84110598600667, 0.8700558991883418, 1.1556301815058487, 1.0558057877237697, 0.9593441405966971, 0.94860955894161, 1.1637725041314626, 1.0815711321728936, 1.0583492052405916, 0.9627433056618342, 1.3670140359267535, 1.2705709383637194, 1.1622428458162537, 1.0675745501188607, 1.1957367214693495, 1.1112711965314517, 0.9959768067004482, 0.903076335509746, 0.8931558925185026, 0.7912932590255418, 0.7799621982699588, 0.6868562778940714, 0.5876192160650839, 0.5636586250712405, 0.5509064805128926, 0.5584410071021524 ], "peak_layer": 1, "peak_normalized": 2.337533863407778, "population_cv": 0.31926473671477396, "rest_mean": 8.519499951270754e-05, "spike_contrast": 1.023604833225597, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.720601326785981e-05, "top_five_layers": [ 1, 17, 18, 21, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.00019988069834653288, 9.591288835508749e-05, 7.95691812527366e-05, 6.706596468575299e-05, 0.00010114281030837446, 9.070643136510625e-05, 7.192231714725494e-05, 7.439780165441334e-05, 9.88170359050855e-05, 9.028112981468439e-05, 8.203276956919581e-05, 8.111486386042088e-05, 9.951327956514433e-05, 9.248430433217436e-05, 9.049861546372995e-05, 8.23234295239672e-05, 0.00011689230450429022, 0.00010864553041756153, 9.938247967511415e-05, 9.12874675123021e-05, 0.00010224651487078518, 9.50239336816594e-05, 8.516520028933883e-05, 7.722135342191905e-05, 7.637306407559663e-05, 6.766286969650537e-05, 6.669395952485502e-05, 5.873254485777579e-05, 5.024686106480658e-05, 4.819800960831344e-05, 4.7107583668548614e-05, 4.775185516336933e-05 ] } } } } } }, "step": 500 }, { "modes": { "learned": { "bits_per_byte": 2.0533380169036586, "forward": { "activation_sha256": { "attention_branch_output": "9fd6aaea3d231a1caa717e4fc44c63b6ac0b7fcac0845e349ac60b57e28fd322", "mlp_branch_output": "884b74f81575660d3a5ef373d83980c2b6a40a5e790421e3f7efc3bf3cdbb781", "post_attention_state": "2d82179ff03320f5589762540fc93ada0edba22cabc07ab3982cf783bff4a116", "post_mlp_state": "b054da1b5190b8b19869952768070eae883b70fbd71c9a2eaf9205894fe44b7f", "pre_attention_input": "4d1314009c174d2647fbc47c0cd287abca38783c7de30eced47b74a084c87fbc", "pre_mlp_input": "fbd2d69d0c904dc853dab2fd491c5cf23fa8369610dbace5a1307601df767adc" }, "logits_sha256": "6387ffea973321ec6422be9fe114fef42d7afbe7d0316fcff56991b4114eed58", "loss_nats": 1.4232654571533203, "mixer_summary_sha256": "483552b4d1a3270c8696995d30bd808a9918e953c1f6ea772d07f8deb85d4145" }, "loss_nats": 1.4232654571533203, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.39672940969467163, "group": 1, "latest_source_mass": 0.1456991732120514, "layer": 1, "max_source_mass": 0.854300856590271, "mixer_index": 1, "normalized_entropy": 0.5723595519413086, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.854300856590271, "median": 0.8727816939353943, "p05": 0.7123822569847107, "p95": 0.9339146018028259 }, { "label": "current_group_partial", "mean": 0.1456991732120514, "median": 0.12721827626228333, "p05": 0.06608537584543228, "p95": 0.2876177430152893 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6258258819580078, "group": 1, "latest_source_mass": 0.40511131286621094, "layer": 2, "max_source_mass": 0.5948886871337891, "mixer_index": 2, "normalized_entropy": 0.9028758963607797, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5948886871337891, "median": 0.5920770168304443, "p05": 0.33635830879211426, "p95": 0.8293883800506592 }, { "label": "current_group_partial", "mean": 0.40511131286621094, "median": 0.4079229235649109, "p05": 0.17061161994934082, "p95": 0.6636416912078857 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5053832530975342, "group": 1, "latest_source_mass": 0.221052348613739, "layer": 2, "max_source_mass": 0.778947651386261, "mixer_index": 3, "normalized_entropy": 0.7291139129921445, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.778947651386261, "median": 0.8033280372619629, "p05": 0.6048110723495483, "p95": 0.8962967395782471 }, { "label": "current_group_partial", "mean": 0.221052348613739, "median": 0.19667191803455353, "p05": 0.10370326042175293, "p95": 0.39518892765045166 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6737473011016846, "group": 1, "latest_source_mass": 0.45933952927589417, "layer": 3, "max_source_mass": 0.5406605005264282, "mixer_index": 4, "normalized_entropy": 0.9720118901117236, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5406605005264282, "median": 0.5336101055145264, "p05": 0.410135954618454, "p95": 0.6916014552116394 }, { "label": "current_group_partial", "mean": 0.45933952927589417, "median": 0.46638989448547363, "p05": 0.3083984851837158, "p95": 0.5898640155792236 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6529731750488281, "group": 1, "latest_source_mass": 0.43803298473358154, "layer": 3, "max_source_mass": 0.5619670152664185, "mixer_index": 5, "normalized_entropy": 0.9420411614764654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5619670152664185, "median": 0.5604932308197021, "p05": 0.3686542510986328, "p95": 0.7799927592277527 }, { "label": "current_group_partial", "mean": 0.43803298473358154, "median": 0.43950676918029785, "p05": 0.22000724077224731, "p95": 0.6313457489013672 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6891955733299255, "group": 1, "latest_source_mass": 0.49001744389533997, "layer": 4, "max_source_mass": 0.5099825859069824, "mixer_index": 6, "normalized_entropy": 0.9942990358457096, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5099825859069824, "median": 0.5090322494506836, "p05": 0.44019854068756104, "p95": 0.5823723077774048 }, { "label": "current_group_partial", "mean": 0.49001744389533997, "median": 0.4909677803516388, "p05": 0.4176277220249176, "p95": 0.559801459312439 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.48565131425857544, "group": 1, "latest_source_mass": 0.20070326328277588, "layer": 4, "max_source_mass": 0.7992967367172241, "mixer_index": 7, "normalized_entropy": 0.7006467426820543, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7992967367172241, "median": 0.8062431216239929, "p05": 0.6658452153205872, "p95": 0.9043658375740051 }, { "label": "current_group_partial", "mean": 0.20070326328277588, "median": 0.1937568634748459, "p05": 0.09563417732715607, "p95": 0.33415475487709045 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6875144243240356, "group": 2, "latest_source_mass": 0.4808617830276489, "layer": 5, "max_source_mass": 0.5191382169723511, "mixer_index": 8, "normalized_entropy": 0.9918736505119168, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5191382169723511, "median": 0.5118080973625183, "p05": 0.45191603899002075, "p95": 0.6115704774856567 }, { "label": "completed_group_1", "mean": 0.4808617830276489, "median": 0.4881919026374817, "p05": 0.38842952251434326, "p95": 0.5480839610099792 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.048142671585083, "group": 2, "latest_source_mass": 0.4353799819946289, "layer": 5, "max_source_mass": 0.4353799819946289, "mixer_index": 9, "normalized_entropy": 0.9540605747781931, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3059995770454407, "median": 0.28220513463020325, "p05": 0.191843643784523, "p95": 0.5426149368286133 }, { "label": "completed_group_1", "mean": 0.25862041115760803, "median": 0.2592811584472656, "p05": 0.1660565435886383, "p95": 0.3212765157222748 }, { "label": "current_group_partial", "mean": 0.4353799819946289, "median": 0.44094622135162354, "p05": 0.28141430020332336, "p95": 0.5360540151596069 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.078475832939148, "group": 2, "latest_source_mass": 0.41806069016456604, "layer": 6, "max_source_mass": 0.41806069016456604, "mixer_index": 10, "normalized_entropy": 0.9816710081102642, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2919672429561615, "median": 0.28502336144447327, "p05": 0.241220623254776, "p95": 0.36694014072418213 }, { "label": "completed_group_1", "mean": 0.28997206687927246, "median": 0.2892274856567383, "p05": 0.2705146372318268, "p95": 0.3101809024810791 }, { "label": "current_group_partial", "mean": 0.41806069016456604, "median": 0.42803525924682617, "p05": 0.32804495096206665, "p95": 0.465151309967041 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0040717124938965, "group": 2, "latest_source_mass": 0.5026139616966248, "layer": 6, "max_source_mass": 0.5026139616966248, "mixer_index": 11, "normalized_entropy": 0.9139454590583285, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.27878478169441223, "median": 0.24224042892456055, "p05": 0.1698543280363083, "p95": 0.5158803462982178 }, { "label": "completed_group_1", "mean": 0.21860124170780182, "median": 0.21914872527122498, "p05": 0.14686164259910583, "p95": 0.26903262734413147 }, { "label": "current_group_partial", "mean": 0.5026139616966248, "median": 0.531975269317627, "p05": 0.31593379378318787, "p95": 0.5875058770179749 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0727062225341797, "group": 2, "latest_source_mass": 0.42498481273651123, "layer": 7, "max_source_mass": 0.42498481273651123, "mixer_index": 12, "normalized_entropy": 0.9764192823973078, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3306647837162018, "median": 0.3299825191497803, "p05": 0.30597272515296936, "p95": 0.35778582096099854 }, { "label": "completed_group_1", "mean": 0.2443503737449646, "median": 0.24191564321517944, "p05": 0.2250334620475769, "p95": 0.27701011300086975 }, { "label": "current_group_partial", "mean": 0.42498481273651123, "median": 0.4262530505657196, "p05": 0.3939187526702881, "p95": 0.44790196418762207 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.036304235458374, "group": 2, "latest_source_mass": 0.455292671918869, "layer": 7, "max_source_mass": 0.455292671918869, "mixer_index": 13, "normalized_entropy": 0.9432847658337463, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3365095853805542, "median": 0.320411741733551, "p05": 0.24313315749168396, "p95": 0.4848058521747589 }, { "label": "completed_group_1", "mean": 0.20819777250289917, "median": 0.20705200731754303, "p05": 0.16223153471946716, "p95": 0.2605344355106354 }, { "label": "current_group_partial", "mean": 0.455292671918869, "median": 0.4683176279067993, "p05": 0.3228694796562195, "p95": 0.5329623222351074 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.071068525314331, "group": 2, "latest_source_mass": 0.42717796564102173, "layer": 8, "max_source_mass": 0.42717796564102173, "mixer_index": 14, "normalized_entropy": 0.9749285861464638, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.32915356755256653, "median": 0.3258785605430603, "p05": 0.2938454747200012, "p95": 0.37063664197921753 }, { "label": "completed_group_1", "mean": 0.24366845190525055, "median": 0.24299904704093933, "p05": 0.21890395879745483, "p95": 0.2733282744884491 }, { "label": "current_group_partial", "mean": 0.42717796564102173, "median": 0.42829641699790955, "p05": 0.3896912634372711, "p95": 0.45910394191741943 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0272725820541382, "group": 2, "latest_source_mass": 0.48878318071365356, "layer": 8, "max_source_mass": 0.48878318071365356, "mixer_index": 15, "normalized_entropy": 0.935063800623913, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29237720370292664, "median": 0.2755309045314789, "p05": 0.21469420194625854, "p95": 0.42577075958251953 }, { "label": "completed_group_1", "mean": 0.2188395857810974, "median": 0.21017195284366608, "p05": 0.15934023261070251, "p95": 0.29730042815208435 }, { "label": "current_group_partial", "mean": 0.48878318071365356, "median": 0.4994751214981079, "p05": 0.38404613733291626, "p95": 0.5597336292266846 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0856674909591675, "group": 3, "latest_source_mass": 0.35679009556770325, "layer": 9, "max_source_mass": 0.3748946189880371, "mixer_index": 16, "normalized_entropy": 0.9882171373445715, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3748946189880371, "median": 0.37628722190856934, "p05": 0.32393988966941833, "p95": 0.42887312173843384 }, { "label": "completed_group_1", "mean": 0.26831525564193726, "median": 0.2661568522453308, "p05": 0.2455824464559555, "p95": 0.29672208428382874 }, { "label": "completed_group_2", "mean": 0.35679009556770325, "median": 0.3568301498889923, "p05": 0.3128414750099182, "p95": 0.39937278628349304 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3326265811920166, "group": 3, "latest_source_mass": 0.3317619562149048, "layer": 9, "max_source_mass": 0.3317619562149048, "mixer_index": 17, "normalized_entropy": 0.9612868800212679, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.26391589641571045, "median": 0.2413073182106018, "p05": 0.17984914779663086, "p95": 0.4526640474796295 }, { "label": "completed_group_1", "mean": 0.14978256821632385, "median": 0.14740070700645447, "p05": 0.10416575521230698, "p95": 0.1980961263179779 }, { "label": "completed_group_2", "mean": 0.2545396089553833, "median": 0.2590503394603729, "p05": 0.16330178081989288, "p95": 0.31726187467575073 }, { "label": "current_group_partial", "mean": 0.3317619562149048, "median": 0.33179306983947754, "p05": 0.2672363519668579, "p95": 0.39366772770881653 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3771977424621582, "group": 3, "latest_source_mass": 0.2781716585159302, "layer": 10, "max_source_mass": 0.2781716585159302, "mixer_index": 18, "normalized_entropy": 0.9934381766868158, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.27525436878204346, "median": 0.27489086985588074, "p05": 0.24440184235572815, "p95": 0.3085845708847046 }, { "label": "completed_group_1", "mean": 0.20677447319030762, "median": 0.20647433400154114, "p05": 0.1837204098701477, "p95": 0.23251454532146454 }, { "label": "completed_group_2", "mean": 0.23979949951171875, "median": 0.24061384797096252, "p05": 0.20753827691078186, "p95": 0.26440829038619995 }, { "label": "current_group_partial", "mean": 0.2781716585159302, "median": 0.27804386615753174, "p05": 0.25431159138679504, "p95": 0.3010777235031128 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3576545715332031, "group": 3, "latest_source_mass": 0.3106929659843445, "layer": 10, "max_source_mass": 0.3106929659843445, "mixer_index": 19, "normalized_entropy": 0.9793407587955913, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2848266065120697, "median": 0.28166642785072327, "p05": 0.22695904970169067, "p95": 0.35531073808670044 }, { "label": "completed_group_1", "mean": 0.17341844737529755, "median": 0.17138710618019104, "p05": 0.14481288194656372, "p95": 0.20774812996387482 }, { "label": "completed_group_2", "mean": 0.23106196522712708, "median": 0.23154035210609436, "p05": 0.18570835888385773, "p95": 0.2726413607597351 }, { "label": "current_group_partial", "mean": 0.3106929659843445, "median": 0.3093051314353943, "p05": 0.2755521535873413, "p95": 0.35260096192359924 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3772028684616089, "group": 3, "latest_source_mass": 0.2857057750225067, "layer": 11, "max_source_mass": 0.2857057750225067, "mixer_index": 20, "normalized_entropy": 0.9934418743138093, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2702856659889221, "median": 0.2702763080596924, "p05": 0.24336200952529907, "p95": 0.2981995642185211 }, { "label": "completed_group_1", "mean": 0.21318203210830688, "median": 0.21185705065727234, "p05": 0.18305814266204834, "p95": 0.24470563232898712 }, { "label": "completed_group_2", "mean": 0.23082655668258667, "median": 0.23429268598556519, "p05": 0.19758956134319305, "p95": 0.24867244064807892 }, { "label": "current_group_partial", "mean": 0.2857057750225067, "median": 0.2861127257347107, "p05": 0.25794482231140137, "p95": 0.31186288595199585 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3277652263641357, "group": 3, "latest_source_mass": 0.3302627205848694, "layer": 11, "max_source_mass": 0.3302627205848694, "mixer_index": 21, "normalized_entropy": 0.9577801537701753, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.31198394298553467, "median": 0.3185040354728699, "p05": 0.1839737594127655, "p95": 0.41595304012298584 }, { "label": "completed_group_1", "mean": 0.16646918654441833, "median": 0.16503849625587463, "p05": 0.13032540678977966, "p95": 0.20871388912200928 }, { "label": "completed_group_2", "mean": 0.1912841945886612, "median": 0.1874908059835434, "p05": 0.13776838779449463, "p95": 0.26046234369277954 }, { "label": "current_group_partial", "mean": 0.3302627205848694, "median": 0.32645535469055176, "p05": 0.2624559700489044, "p95": 0.407662570476532 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3750126361846924, "group": 3, "latest_source_mass": 0.2897627055644989, "layer": 12, "max_source_mass": 0.2897627055644989, "mixer_index": 22, "normalized_entropy": 0.9918619556916581, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2621780037879944, "median": 0.2640007734298706, "p05": 0.212703138589859, "p95": 0.3065672814846039 }, { "label": "completed_group_1", "mean": 0.21051618456840515, "median": 0.20835110545158386, "p05": 0.18514882028102875, "p95": 0.24316135048866272 }, { "label": "completed_group_2", "mean": 0.23754313588142395, "median": 0.23885108530521393, "p05": 0.2025136798620224, "p95": 0.26575183868408203 }, { "label": "current_group_partial", "mean": 0.2897627055644989, "median": 0.2845831513404846, "p05": 0.25373271107673645, "p95": 0.3410085439682007 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3568847179412842, "group": 3, "latest_source_mass": 0.3233448266983032, "layer": 12, "max_source_mass": 0.3233448266983032, "mixer_index": 23, "normalized_entropy": 0.9787854268159553, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2628105878829956, "median": 0.26141154766082764, "p05": 0.18301242589950562, "p95": 0.35439273715019226 }, { "label": "completed_group_1", "mean": 0.1954234391450882, "median": 0.1943855881690979, "p05": 0.16116495430469513, "p95": 0.2287053018808365 }, { "label": "completed_group_2", "mean": 0.21842113137245178, "median": 0.21915890276432037, "p05": 0.18287011981010437, "p95": 0.2523847222328186 }, { "label": "current_group_partial", "mean": 0.3233448266983032, "median": 0.3182007968425751, "p05": 0.26236361265182495, "p95": 0.4023315906524658 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3733241558074951, "group": 4, "latest_source_mass": 0.2984467148780823, "layer": 13, "max_source_mass": 0.2984467148780823, "mixer_index": 24, "normalized_entropy": 0.9906439745582477, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25842779874801636, "median": 0.2585429549217224, "p05": 0.2317616194486618, "p95": 0.28543001413345337 }, { "label": "completed_group_1", "mean": 0.22964224219322205, "median": 0.22467529773712158, "p05": 0.20248907804489136, "p95": 0.27881020307540894 }, { "label": "completed_group_2", "mean": 0.21348324418067932, "median": 0.2143072485923767, "p05": 0.18921855092048645, "p95": 0.23720777034759521 }, { "label": "completed_group_3", "mean": 0.2984467148780823, "median": 0.30472591519355774, "p05": 0.22567860782146454, "p95": 0.35220274329185486 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5855276584625244, "group": 4, "latest_source_mass": 0.2245527058839798, "layer": 13, "max_source_mass": 0.2499181628227234, "mixer_index": 25, "normalized_entropy": 0.9851437239132672, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.18060272932052612, "median": 0.17444205284118652, "p05": 0.12549439072608948, "p95": 0.27444836497306824 }, { "label": "completed_group_1", "mean": 0.1563272476196289, "median": 0.15402349829673767, "p05": 0.1285230815410614, "p95": 0.19161051511764526 }, { "label": "completed_group_2", "mean": 0.1885991394519806, "median": 0.19113421440124512, "p05": 0.1411750763654709, "p95": 0.22396130859851837 }, { "label": "completed_group_3", "mean": 0.2499181628227234, "median": 0.25040146708488464, "p05": 0.20360784232616425, "p95": 0.2959936261177063 }, { "label": "current_group_partial", "mean": 0.2245527058839798, "median": 0.22453144192695618, "p05": 0.1881018728017807, "p95": 0.2609967589378357 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5906484127044678, "group": 4, "latest_source_mass": 0.2668720781803131, "layer": 14, "max_source_mass": 0.2668720781803131, "mixer_index": 26, "normalized_entropy": 0.9883254274150809, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19773373007774353, "median": 0.1965319812297821, "p05": 0.18224859237670898, "p95": 0.2179831713438034 }, { "label": "completed_group_1", "mean": 0.18790484964847565, "median": 0.18465439975261688, "p05": 0.17162445187568665, "p95": 0.21807827055454254 }, { "label": "completed_group_2", "mean": 0.15935488045215607, "median": 0.15496674180030823, "p05": 0.1365157663822174, "p95": 0.1937219798564911 }, { "label": "completed_group_3", "mean": 0.18813443183898926, "median": 0.18904224038124084, "p05": 0.15670713782310486, "p95": 0.21698647737503052 }, { "label": "current_group_partial", "mean": 0.2668720781803131, "median": 0.2718026638031006, "p05": 0.22497205436229706, "p95": 0.2880238890647888 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5916568040847778, "group": 4, "latest_source_mass": 0.22534610331058502, "layer": 14, "max_source_mass": 0.227544903755188, "mixer_index": 27, "normalized_entropy": 0.9889519762073764, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2148856818675995, "median": 0.21614974737167358, "p05": 0.17341811954975128, "p95": 0.2519548237323761 }, { "label": "completed_group_1", "mean": 0.14808836579322815, "median": 0.1478005349636078, "p05": 0.1207289919257164, "p95": 0.17668096721172333 }, { "label": "completed_group_2", "mean": 0.18413493037223816, "median": 0.18671013414859772, "p05": 0.1423187255859375, "p95": 0.21475675702095032 }, { "label": "completed_group_3", "mean": 0.227544903755188, "median": 0.22678817808628082, "p05": 0.19582602381706238, "p95": 0.25986844301223755 }, { "label": "current_group_partial", "mean": 0.22534610331058502, "median": 0.22619590163230896, "p05": 0.18310406804084778, "p95": 0.2634280323982239 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5474352836608887, "group": 4, "latest_source_mass": 0.31912651658058167, "layer": 15, "max_source_mass": 0.31912651658058167, "mixer_index": 28, "normalized_entropy": 0.9614756007086727, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.19825294613838196, "median": 0.19680598378181458, "p05": 0.17132307589054108, "p95": 0.2297736555337906 }, { "label": "completed_group_1", "mean": 0.17294847965240479, "median": 0.164862722158432, "p05": 0.14083626866340637, "p95": 0.2345556914806366 }, { "label": "completed_group_2", "mean": 0.11353601515293121, "median": 0.10715661942958832, "p05": 0.0958808958530426, "p95": 0.15557381510734558 }, { "label": "completed_group_3", "mean": 0.19613605737686157, "median": 0.19778506457805634, "p05": 0.15166249871253967, "p95": 0.23660960793495178 }, { "label": "current_group_partial", "mean": 0.31912651658058167, "median": 0.327135294675827, "p05": 0.24559810757637024, "p95": 0.3559069037437439 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5919160842895508, "group": 4, "latest_source_mass": 0.2247331142425537, "layer": 15, "max_source_mass": 0.2257765829563141, "mixer_index": 29, "normalized_entropy": 0.9891130760564416, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21136541664600372, "median": 0.2003786563873291, "p05": 0.1631811559200287, "p95": 0.28519296646118164 }, { "label": "completed_group_1", "mean": 0.1572093367576599, "median": 0.15547874569892883, "p05": 0.12932342290878296, "p95": 0.1883375495672226 }, { "label": "completed_group_2", "mean": 0.18091553449630737, "median": 0.18437424302101135, "p05": 0.14374977350234985, "p95": 0.20782355964183807 }, { "label": "completed_group_3", "mean": 0.2257765829563141, "median": 0.22509169578552246, "p05": 0.18708458542823792, "p95": 0.2700234353542328 }, { "label": "current_group_partial", "mean": 0.2247331142425537, "median": 0.22534477710723877, "p05": 0.18621239066123962, "p95": 0.261106938123703 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.542572021484375, "group": 4, "latest_source_mass": 0.32280486822128296, "layer": 16, "max_source_mass": 0.32280486822128296, "mixer_index": 30, "normalized_entropy": 0.9584538860224823, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2053474634885788, "median": 0.20845530927181244, "p05": 0.1648181527853012, "p95": 0.24006053805351257 }, { "label": "completed_group_1", "mean": 0.19789552688598633, "median": 0.19339492917060852, "p05": 0.17084503173828125, "p95": 0.24170872569084167 }, { "label": "completed_group_2", "mean": 0.10896679013967514, "median": 0.10333000868558884, "p05": 0.09237737953662872, "p95": 0.14624080061912537 }, { "label": "completed_group_3", "mean": 0.16498532891273499, "median": 0.16568085551261902, "p05": 0.12339095771312714, "p95": 0.20368334650993347 }, { "label": "current_group_partial", "mean": 0.32280486822128296, "median": 0.3227658271789551, "p05": 0.27662473917007446, "p95": 0.37092989683151245 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5774884223937988, "group": 4, "latest_source_mass": 0.24561989307403564, "layer": 16, "max_source_mass": 0.24561989307403564, "mixer_index": 31, "normalized_entropy": 0.9801486656965963, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1874420940876007, "median": 0.1747647374868393, "p05": 0.12934981286525726, "p95": 0.3250933289527893 }, { "label": "completed_group_1", "mean": 0.14777487516403198, "median": 0.1479286551475525, "p05": 0.11702761054039001, "p95": 0.18031248450279236 }, { "label": "completed_group_2", "mean": 0.18392466008663177, "median": 0.1873953938484192, "p05": 0.13232499361038208, "p95": 0.21972279250621796 }, { "label": "completed_group_3", "mean": 0.23523849248886108, "median": 0.23474374413490295, "p05": 0.18369318544864655, "p95": 0.28937381505966187 }, { "label": "current_group_partial", "mean": 0.24561989307403564, "median": 0.24738794565200806, "p05": 0.18173503875732422, "p95": 0.3010544776916504 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.585869312286377, "group": 5, "latest_source_mass": 0.26391667127609253, "layer": 17, "max_source_mass": 0.26391667127609253, "mixer_index": 32, "normalized_entropy": 0.9853560053695527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2123720794916153, "median": 0.21364213526248932, "p05": 0.18289057910442352, "p95": 0.24095472693443298 }, { "label": "completed_group_1", "mean": 0.19153766334056854, "median": 0.19043566286563873, "p05": 0.16703367233276367, "p95": 0.22240948677062988 }, { "label": "completed_group_2", "mean": 0.14032581448554993, "median": 0.1387615203857422, "p05": 0.1285465657711029, "p95": 0.15656951069831848 }, { "label": "completed_group_3", "mean": 0.1918477714061737, "median": 0.19085454940795898, "p05": 0.15591931343078613, "p95": 0.22882042825222015 }, { "label": "completed_group_4", "mean": 0.26391667127609253, "median": 0.26347947120666504, "p05": 0.23446886241436005, "p95": 0.29670727252960205 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.747511625289917, "group": 5, "latest_source_mass": 0.2640940546989441, "layer": 17, "max_source_mass": 0.2640940546989441, "mixer_index": 33, "normalized_entropy": 0.9753048080961441, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13209694623947144, "median": 0.12275595963001251, "p05": 0.09756084531545639, "p95": 0.2136262208223343 }, { "label": "completed_group_1", "mean": 0.11971477419137955, "median": 0.11888641864061356, "p05": 0.10256937146186829, "p95": 0.1397632211446762 }, { "label": "completed_group_2", "mean": 0.14703533053398132, "median": 0.1483881175518036, "p05": 0.12076307833194733, "p95": 0.1677870750427246 }, { "label": "completed_group_3", "mean": 0.17972922325134277, "median": 0.17958298325538635, "p05": 0.1569097489118576, "p95": 0.20113393664360046 }, { "label": "completed_group_4", "mean": 0.15732964873313904, "median": 0.15857598185539246, "p05": 0.12811288237571716, "p95": 0.18277260661125183 }, { "label": "current_group_partial", "mean": 0.2640940546989441, "median": 0.2643871307373047, "p05": 0.23343844711780548, "p95": 0.29374754428863525 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6791677474975586, "group": 5, "latest_source_mass": 0.3440830111503601, "layer": 18, "max_source_mass": 0.3440830111503601, "mixer_index": 34, "normalized_entropy": 0.937161363640509, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1429731547832489, "median": 0.1472356617450714, "p05": 0.10260166227817535, "p95": 0.18026825785636902 }, { "label": "completed_group_1", "mean": 0.128341943025589, "median": 0.1243518516421318, "p05": 0.10659048706293106, "p95": 0.16665902733802795 }, { "label": "completed_group_2", "mean": 0.08682706952095032, "median": 0.08546946942806244, "p05": 0.07587063312530518, "p95": 0.10337291657924652 }, { "label": "completed_group_3", "mean": 0.12974600493907928, "median": 0.12982705235481262, "p05": 0.10628668963909149, "p95": 0.15255016088485718 }, { "label": "completed_group_4", "mean": 0.16802877187728882, "median": 0.16956090927124023, "p05": 0.13419006764888763, "p95": 0.19487163424491882 }, { "label": "current_group_partial", "mean": 0.3440830111503601, "median": 0.3435801863670349, "p05": 0.3040219247341156, "p95": 0.3866944909095764 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7079570293426514, "group": 5, "latest_source_mass": 0.3138047456741333, "layer": 18, "max_source_mass": 0.3138047456741333, "mixer_index": 35, "normalized_entropy": 0.9532289677690342, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12207645177841187, "median": 0.11115410923957825, "p05": 0.08012475818395615, "p95": 0.22810664772987366 }, { "label": "completed_group_1", "mean": 0.11454500257968903, "median": 0.11326663196086884, "p05": 0.09493672847747803, "p95": 0.13798359036445618 }, { "label": "completed_group_2", "mean": 0.14774823188781738, "median": 0.14947353303432465, "p05": 0.11176952719688416, "p95": 0.17675937712192535 }, { "label": "completed_group_3", "mean": 0.1675407588481903, "median": 0.16761797666549683, "p05": 0.13766297698020935, "p95": 0.19680744409561157 }, { "label": "completed_group_4", "mean": 0.1342848241329193, "median": 0.13416096568107605, "p05": 0.10613290220499039, "p95": 0.1641635000705719 }, { "label": "current_group_partial", "mean": 0.3138047456741333, "median": 0.31486788392066956, "p05": 0.2550574839115143, "p95": 0.36806803941726685 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.689455270767212, "group": 5, "latest_source_mass": 0.33726364374160767, "layer": 19, "max_source_mass": 0.33726364374160767, "mixer_index": 36, "normalized_entropy": 0.9429029396981957, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14791089296340942, "median": 0.15329936146736145, "p05": 0.09563843905925751, "p95": 0.19142045080661774 }, { "label": "completed_group_1", "mean": 0.15263086557388306, "median": 0.15167269110679626, "p05": 0.13290324807167053, "p95": 0.17540162801742554 }, { "label": "completed_group_2", "mean": 0.09421356767416, "median": 0.0932738184928894, "p05": 0.0817713513970375, "p95": 0.10977791249752045 }, { "label": "completed_group_3", "mean": 0.12123478949069977, "median": 0.12004932761192322, "p05": 0.10300692170858383, "p95": 0.1423986405134201 }, { "label": "completed_group_4", "mean": 0.14674627780914307, "median": 0.14631736278533936, "p05": 0.12582728266716003, "p95": 0.1688128113746643 }, { "label": "current_group_partial", "mean": 0.33726364374160767, "median": 0.33654576539993286, "p05": 0.28587767481803894, "p95": 0.3911621868610382 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6673545837402344, "group": 5, "latest_source_mass": 0.3504054546356201, "layer": 19, "max_source_mass": 0.3504054546356201, "mixer_index": 37, "normalized_entropy": 0.9305683114143563, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.119766965508461, "median": 0.10288102924823761, "p05": 0.06773775815963745, "p95": 0.26150354743003845 }, { "label": "completed_group_1", "mean": 0.10584045946598053, "median": 0.10460227727890015, "p05": 0.08871804922819138, "p95": 0.12676651775836945 }, { "label": "completed_group_2", "mean": 0.1391725242137909, "median": 0.13989759981632233, "p05": 0.10571101307868958, "p95": 0.16792798042297363 }, { "label": "completed_group_3", "mean": 0.15867863595485687, "median": 0.15944495797157288, "p05": 0.11882050335407257, "p95": 0.19538073241710663 }, { "label": "completed_group_4", "mean": 0.1261359602212906, "median": 0.1259205937385559, "p05": 0.09519180655479431, "p95": 0.15761253237724304 }, { "label": "current_group_partial", "mean": 0.3504054546356201, "median": 0.350231796503067, "p05": 0.2736193835735321, "p95": 0.4254133105278015 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7276721000671387, "group": 5, "latest_source_mass": 0.2906533479690552, "layer": 20, "max_source_mass": 0.2906533479690552, "mixer_index": 38, "normalized_entropy": 0.9642321582435799, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16197484731674194, "median": 0.16698119044303894, "p05": 0.10772223770618439, "p95": 0.20956003665924072 }, { "label": "completed_group_1", "mean": 0.16130127012729645, "median": 0.15952633321285248, "p05": 0.1394604593515396, "p95": 0.18821176886558533 }, { "label": "completed_group_2", "mean": 0.1033584251999855, "median": 0.10190790891647339, "p05": 0.08996254205703735, "p95": 0.12321754544973373 }, { "label": "completed_group_3", "mean": 0.12991489470005035, "median": 0.1302758753299713, "p05": 0.10520754754543304, "p95": 0.15511953830718994 }, { "label": "completed_group_4", "mean": 0.15279719233512878, "median": 0.15249209105968475, "p05": 0.1319577395915985, "p95": 0.17580842971801758 }, { "label": "current_group_partial", "mean": 0.2906533479690552, "median": 0.28811272978782654, "p05": 0.24316680431365967, "p95": 0.34837183356285095 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.655259609222412, "group": 5, "latest_source_mass": 0.34562793374061584, "layer": 20, "max_source_mass": 0.34562793374061584, "mixer_index": 39, "normalized_entropy": 0.9238179776080931, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12291314452886581, "median": 0.10247613489627838, "p05": 0.04299236834049225, "p95": 0.3265020251274109 }, { "label": "completed_group_1", "mean": 0.10535868257284164, "median": 0.10351927578449249, "p05": 0.08489269018173218, "p95": 0.13092145323753357 }, { "label": "completed_group_2", "mean": 0.14431552588939667, "median": 0.1464514434337616, "p05": 0.1031925305724144, "p95": 0.17885252833366394 }, { "label": "completed_group_3", "mean": 0.1644165813922882, "median": 0.16633014380931854, "p05": 0.11522950232028961, "p95": 0.20475128293037415 }, { "label": "completed_group_4", "mean": 0.11736814677715302, "median": 0.11602836102247238, "p05": 0.08080777525901794, "p95": 0.15889662504196167 }, { "label": "current_group_partial", "mean": 0.34562793374061584, "median": 0.3455731272697449, "p05": 0.2439023107290268, "p95": 0.4484919309616089 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7456457614898682, "group": 6, "latest_source_mass": 0.2435421347618103, "layer": 21, "max_source_mass": 0.2435421347618103, "mixer_index": 40, "normalized_entropy": 0.9742634496816395, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17144063115119934, "median": 0.17784935235977173, "p05": 0.11719495803117752, "p95": 0.2109185755252838 }, { "label": "completed_group_1", "mean": 0.17272350192070007, "median": 0.17066533863544464, "p05": 0.15510167181491852, "p95": 0.19745397567749023 }, { "label": "completed_group_2", "mean": 0.09372986108064651, "median": 0.09023232758045197, "p05": 0.0791064128279686, "p95": 0.11955329030752182 }, { "label": "completed_group_3", "mean": 0.13482090830802917, "median": 0.1349901258945465, "p05": 0.10589948296546936, "p95": 0.16432839632034302 }, { "label": "completed_group_4", "mean": 0.183742955327034, "median": 0.18362680077552795, "p05": 0.16126227378845215, "p95": 0.2063293308019638 }, { "label": "completed_group_5", "mean": 0.2435421347618103, "median": 0.2420862764120102, "p05": 0.20024174451828003, "p95": 0.29210829734802246 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.8196223974227905, "group": 6, "latest_source_mass": 0.27700191736221313, "layer": 21, "max_source_mass": 0.27700191736221313, "mixer_index": 41, "normalized_entropy": 0.9351009337744438, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0976112112402916, "median": 0.083127960562706, "p05": 0.057163339108228683, "p95": 0.21396708488464355 }, { "label": "completed_group_1", "mean": 0.08662202209234238, "median": 0.0854487419128418, "p05": 0.06999586522579193, "p95": 0.10710393637418747 }, { "label": "completed_group_2", "mean": 0.10460365563631058, "median": 0.10444506257772446, "p05": 0.08147601783275604, "p95": 0.12855757772922516 }, { "label": "completed_group_3", "mean": 0.12231925129890442, "median": 0.12159790843725204, "p05": 0.09556058049201965, "p95": 0.15222840011119843 }, { "label": "completed_group_4", "mean": 0.09176328033208847, "median": 0.09087449312210083, "p05": 0.06908228248357773, "p95": 0.11806865781545639 }, { "label": "completed_group_5", "mean": 0.22007866203784943, "median": 0.21696066856384277, "p05": 0.16129010915756226, "p95": 0.29291966557502747 }, { "label": "current_group_partial", "mean": 0.27700191736221313, "median": 0.2762490510940552, "p05": 0.20922920107841492, "p95": 0.3463406562805176 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8201665878295898, "group": 6, "latest_source_mass": 0.31871896982192993, "layer": 22, "max_source_mass": 0.31871896982192993, "mixer_index": 42, "normalized_entropy": 0.9353805923224315, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12032122910022736, "median": 0.12179268896579742, "p05": 0.07555080205202103, "p95": 0.17048288881778717 }, { "label": "completed_group_1", "mean": 0.12977322936058044, "median": 0.12879429757595062, "p05": 0.10893770307302475, "p95": 0.15374676883220673 }, { "label": "completed_group_2", "mean": 0.07435770332813263, "median": 0.07368695735931396, "p05": 0.06492579728364944, "p95": 0.08556335419416428 }, { "label": "completed_group_3", "mean": 0.09230788052082062, "median": 0.09124474227428436, "p05": 0.07656185328960419, "p95": 0.11202958226203918 }, { "label": "completed_group_4", "mean": 0.1133352667093277, "median": 0.1122380942106247, "p05": 0.09279464185237885, "p95": 0.138209730386734 }, { "label": "completed_group_5", "mean": 0.15118570625782013, "median": 0.14947570860385895, "p05": 0.11790992319583893, "p95": 0.190475732088089 }, { "label": "current_group_partial", "mean": 0.31871896982192993, "median": 0.3194882273674011, "p05": 0.25439688563346863, "p95": 0.3806491792201996 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7483060359954834, "group": 6, "latest_source_mass": 0.35795146226882935, "layer": 22, "max_source_mass": 0.35795146226882935, "mixer_index": 43, "normalized_entropy": 0.8984515738531086, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09602338075637817, "median": 0.08368823677301407, "p05": 0.048330605030059814, "p95": 0.22025157511234283 }, { "label": "completed_group_1", "mean": 0.07951871305704117, "median": 0.0780613124370575, "p05": 0.06370338052511215, "p95": 0.09924023598432541 }, { "label": "completed_group_2", "mean": 0.09892410784959793, "median": 0.09849533438682556, "p05": 0.07656967639923096, "p95": 0.12212418019771576 }, { "label": "completed_group_3", "mean": 0.11023539304733276, "median": 0.10842189192771912, "p05": 0.08411939442157745, "p95": 0.14120185375213623 }, { "label": "completed_group_4", "mean": 0.08284324407577515, "median": 0.08092047274112701, "p05": 0.05862290784716606, "p95": 0.11340560019016266 }, { "label": "completed_group_5", "mean": 0.17450372874736786, "median": 0.17053596675395966, "p05": 0.10771246254444122, "p95": 0.2589290738105774 }, { "label": "current_group_partial", "mean": 0.35795146226882935, "median": 0.35653698444366455, "p05": 0.2500389516353607, "p95": 0.47316011786460876 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.851926565170288, "group": 6, "latest_source_mass": 0.29032042622566223, "layer": 23, "max_source_mass": 0.29032042622566223, "mixer_index": 44, "normalized_entropy": 0.9517019920315172, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14006078243255615, "median": 0.14251285791397095, "p05": 0.09944885969161987, "p95": 0.17503000795841217 }, { "label": "completed_group_1", "mean": 0.14811867475509644, "median": 0.1468043476343155, "p05": 0.13084301352500916, "p95": 0.16874222457408905 }, { "label": "completed_group_2", "mean": 0.08932579308748245, "median": 0.08844955265522003, "p05": 0.07788823544979095, "p95": 0.10412273555994034 }, { "label": "completed_group_3", "mean": 0.09198777377605438, "median": 0.09086033701896667, "p05": 0.07522772252559662, "p95": 0.11275116354227066 }, { "label": "completed_group_4", "mean": 0.11175549030303955, "median": 0.11076019704341888, "p05": 0.09502518177032471, "p95": 0.1322603076696396 }, { "label": "completed_group_5", "mean": 0.12843109667301178, "median": 0.12729130685329437, "p05": 0.09835763275623322, "p95": 0.16321906447410583 }, { "label": "current_group_partial", "mean": 0.29032042622566223, "median": 0.29004043340682983, "p05": 0.2191990315914154, "p95": 0.3636181354522705 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7523608207702637, "group": 6, "latest_source_mass": 0.3702468276023865, "layer": 23, "max_source_mass": 0.3702468276023865, "mixer_index": 45, "normalized_entropy": 0.9005353210275343, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10785886645317078, "median": 0.10064198076725006, "p05": 0.06490226089954376, "p95": 0.1922605037689209 }, { "label": "completed_group_1", "mean": 0.08607445657253265, "median": 0.08407071977853775, "p05": 0.06460381299257278, "p95": 0.11544128507375717 }, { "label": "completed_group_2", "mean": 0.09595336019992828, "median": 0.09583206474781036, "p05": 0.07447511702775955, "p95": 0.11774584650993347 }, { "label": "completed_group_3", "mean": 0.10649852454662323, "median": 0.10573073476552963, "p05": 0.08023984730243683, "p95": 0.1353880763053894 }, { "label": "completed_group_4", "mean": 0.0838143527507782, "median": 0.08303111791610718, "p05": 0.0620865523815155, "p95": 0.10783524066209793 }, { "label": "completed_group_5", "mean": 0.14955362677574158, "median": 0.14380723237991333, "p05": 0.09627015888690948, "p95": 0.222757488489151 }, { "label": "current_group_partial", "mean": 0.3702468276023865, "median": 0.36859041452407837, "p05": 0.2604178488254547, "p95": 0.483437716960907 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8693451881408691, "group": 6, "latest_source_mass": 0.2815432548522949, "layer": 24, "max_source_mass": 0.2815432548522949, "mixer_index": 46, "normalized_entropy": 0.9606533935024625, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1378144919872284, "median": 0.13807497918605804, "p05": 0.10932769626379013, "p95": 0.16912776231765747 }, { "label": "completed_group_1", "mean": 0.1244172602891922, "median": 0.12220869958400726, "p05": 0.10719747841358185, "p95": 0.14835487306118011 }, { "label": "completed_group_2", "mean": 0.09968579560518265, "median": 0.09896545112133026, "p05": 0.08706579357385635, "p95": 0.11497638374567032 }, { "label": "completed_group_3", "mean": 0.10974415391683578, "median": 0.10904944688081741, "p05": 0.09279274940490723, "p95": 0.1297232210636139 }, { "label": "completed_group_4", "mean": 0.11655184626579285, "median": 0.11616086959838867, "p05": 0.09858430176973343, "p95": 0.13470277190208435 }, { "label": "completed_group_5", "mean": 0.130243182182312, "median": 0.1280980110168457, "p05": 0.09857633709907532, "p95": 0.16894926130771637 }, { "label": "current_group_partial", "mean": 0.2815432548522949, "median": 0.2809160351753235, "p05": 0.21616990864276886, "p95": 0.35214728116989136 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7898914813995361, "group": 6, "latest_source_mass": 0.33616429567337036, "layer": 24, "max_source_mass": 0.33616429567337036, "mixer_index": 47, "normalized_entropy": 0.9198222653129591, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10115307569503784, "median": 0.08613650500774384, "p05": 0.050611600279808044, "p95": 0.23426659405231476 }, { "label": "completed_group_1", "mean": 0.09615936875343323, "median": 0.09461446106433868, "p05": 0.07746638357639313, "p95": 0.11935343593358994 }, { "label": "completed_group_2", "mean": 0.11218561232089996, "median": 0.1115623190999031, "p05": 0.08799588680267334, "p95": 0.13799631595611572 }, { "label": "completed_group_3", "mean": 0.1163020133972168, "median": 0.11555752158164978, "p05": 0.08836641162633896, "p95": 0.14650842547416687 }, { "label": "completed_group_4", "mean": 0.09238721430301666, "median": 0.0899561196565628, "p05": 0.06484603136777878, "p95": 0.1281205415725708 }, { "label": "completed_group_5", "mean": 0.14564841985702515, "median": 0.1386992335319519, "p05": 0.08829846978187561, "p95": 0.22654561698436737 }, { "label": "current_group_partial", "mean": 0.33616429567337036, "median": 0.33723515272140503, "p05": 0.22659176588058472, "p95": 0.4460947513580322 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9094455242156982, "group": 7, "latest_source_mass": 0.22462961077690125, "layer": 25, "max_source_mass": 0.22462961077690125, "mixer_index": 48, "normalized_entropy": 0.981260889739787, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14588291943073273, "median": 0.14661915600299835, "p05": 0.11508320271968842, "p95": 0.1800285279750824 }, { "label": "completed_group_1", "mean": 0.14179158210754395, "median": 0.14040151238441467, "p05": 0.12424254417419434, "p95": 0.16426950693130493 }, { "label": "completed_group_2", "mean": 0.1023377925157547, "median": 0.10112032294273376, "p05": 0.08753453195095062, "p95": 0.12110165506601334 }, { "label": "completed_group_3", "mean": 0.11511939018964767, "median": 0.11405108869075775, "p05": 0.10003029555082321, "p95": 0.13439135253429413 }, { "label": "completed_group_4", "mean": 0.13270339369773865, "median": 0.13297244906425476, "p05": 0.11011375486850739, "p95": 0.15357504785060883 }, { "label": "completed_group_5", "mean": 0.13753530383110046, "median": 0.13631805777549744, "p05": 0.11099010705947876, "p95": 0.16822901368141174 }, { "label": "completed_group_6", "mean": 0.22462961077690125, "median": 0.22316956520080566, "p05": 0.17812120914459229, "p95": 0.27728700637817383 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9082129001617432, "group": 7, "latest_source_mass": 0.2262895107269287, "layer": 25, "max_source_mass": 0.2519546449184418, "mixer_index": 49, "normalized_entropy": 0.9176564293412313, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09145340323448181, "median": 0.09255078434944153, "p05": 0.03969334065914154, "p95": 0.15699249505996704 }, { "label": "completed_group_1", "mean": 0.07698439061641693, "median": 0.07560532540082932, "p05": 0.05995766818523407, "p95": 0.09742964059114456 }, { "label": "completed_group_2", "mean": 0.08525429666042328, "median": 0.08423174172639847, "p05": 0.06901301443576813, "p95": 0.10490919649600983 }, { "label": "completed_group_3", "mean": 0.08438773453235626, "median": 0.08191201090812683, "p05": 0.06093179062008858, "p95": 0.11479861289262772 }, { "label": "completed_group_4", "mean": 0.06885559856891632, "median": 0.06811907887458801, "p05": 0.04831434786319733, "p95": 0.09210654348134995 }, { "label": "completed_group_5", "mean": 0.11482042074203491, "median": 0.10688380897045135, "p05": 0.061463840305805206, "p95": 0.19296644628047943 }, { "label": "completed_group_6", "mean": 0.2519546449184418, "median": 0.250648558139801, "p05": 0.1514165848493576, "p95": 0.35778117179870605 }, { "label": "current_group_partial", "mean": 0.2262895107269287, "median": 0.22269269824028015, "p05": 0.0977732241153717, "p95": 0.37383198738098145 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.025789499282837, "group": 7, "latest_source_mass": 0.20079906284809113, "layer": 26, "max_source_mass": 0.20079906284809113, "mixer_index": 50, "normalized_entropy": 0.9741988215000951, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11047068238258362, "median": 0.11173020303249359, "p05": 0.0543295294046402, "p95": 0.16616837680339813 }, { "label": "completed_group_1", "mean": 0.12069687247276306, "median": 0.11997807025909424, "p05": 0.10084380954504013, "p95": 0.14284764230251312 }, { "label": "completed_group_2", "mean": 0.09073689579963684, "median": 0.08891503512859344, "p05": 0.07266610860824585, "p95": 0.11606847494840622 }, { "label": "completed_group_3", "mean": 0.09690164029598236, "median": 0.09599806368350983, "p05": 0.07936231791973114, "p95": 0.11817526817321777 }, { "label": "completed_group_4", "mean": 0.10987946391105652, "median": 0.10931764543056488, "p05": 0.08768688142299652, "p95": 0.13469792902469635 }, { "label": "completed_group_5", "mean": 0.11353722959756851, "median": 0.1125042587518692, "p05": 0.08439873158931732, "p95": 0.14467501640319824 }, { "label": "completed_group_6", "mean": 0.15697816014289856, "median": 0.15547990798950195, "p05": 0.11792093515396118, "p95": 0.19920390844345093 }, { "label": "current_group_partial", "mean": 0.20079906284809113, "median": 0.19964557886123657, "p05": 0.11376827955245972, "p95": 0.2935184836387634 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.9302079677581787, "group": 7, "latest_source_mass": 0.24795080721378326, "layer": 26, "max_source_mass": 0.24795080721378326, "mixer_index": 51, "normalized_entropy": 0.9282338209896963, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0971018448472023, "median": 0.08827555179595947, "p05": 0.06328719854354858, "p95": 0.159974604845047 }, { "label": "completed_group_1", "mean": 0.08240298926830292, "median": 0.08067642897367477, "p05": 0.06337254494428635, "p95": 0.10580241680145264 }, { "label": "completed_group_2", "mean": 0.0830029845237732, "median": 0.08191139996051788, "p05": 0.06423131376504898, "p95": 0.10520751774311066 }, { "label": "completed_group_3", "mean": 0.08926018327474594, "median": 0.08775374293327332, "p05": 0.06515306234359741, "p95": 0.11704635620117188 }, { "label": "completed_group_4", "mean": 0.07480385899543762, "median": 0.07413309812545776, "p05": 0.055974941700696945, "p95": 0.09636931121349335 }, { "label": "completed_group_5", "mean": 0.10335960239171982, "median": 0.09859646111726761, "p05": 0.0651523768901825, "p95": 0.1582750678062439 }, { "label": "completed_group_6", "mean": 0.22211773693561554, "median": 0.2181653380393982, "p05": 0.12620511651039124, "p95": 0.3312388062477112 }, { "label": "current_group_partial", "mean": 0.24795080721378326, "median": 0.2450115978717804, "p05": 0.12909922003746033, "p95": 0.37674421072006226 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.010532855987549, "group": 7, "latest_source_mass": 0.20431259274482727, "layer": 27, "max_source_mass": 0.20431259274482727, "mixer_index": 52, "normalized_entropy": 0.9668619269591872, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10700424760580063, "median": 0.10718298703432083, "p05": 0.05378019064664841, "p95": 0.16453370451927185 }, { "label": "completed_group_1", "mean": 0.09611140936613083, "median": 0.09571994841098785, "p05": 0.0705467239022255, "p95": 0.1255740523338318 }, { "label": "completed_group_2", "mean": 0.09769980609416962, "median": 0.09758654981851578, "p05": 0.08190305531024933, "p95": 0.11550790816545486 }, { "label": "completed_group_3", "mean": 0.10596007108688354, "median": 0.10523851960897446, "p05": 0.08640100061893463, "p95": 0.12765458226203918 }, { "label": "completed_group_4", "mean": 0.09634587913751602, "median": 0.09565047174692154, "p05": 0.07201918959617615, "p95": 0.12302076816558838 }, { "label": "completed_group_5", "mean": 0.11705268919467926, "median": 0.11422611027956009, "p05": 0.08651825040578842, "p95": 0.15585871040821075 }, { "label": "completed_group_6", "mean": 0.17551329731941223, "median": 0.17338381707668304, "p05": 0.12190233916044235, "p95": 0.23716403543949127 }, { "label": "current_group_partial", "mean": 0.20431259274482727, "median": 0.1999019980430603, "p05": 0.11230528354644775, "p95": 0.3120884299278259 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.9408204555511475, "group": 7, "latest_source_mass": 0.19202858209609985, "layer": 27, "max_source_mass": 0.22555868327617645, "mixer_index": 53, "normalized_entropy": 0.9333373488264999, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08590779453516006, "median": 0.062378786504268646, "p05": 0.03650445491075516, "p95": 0.22098970413208008 }, { "label": "completed_group_1", "mean": 0.09530991315841675, "median": 0.09346900880336761, "p05": 0.06249644607305527, "p95": 0.13247521221637726 }, { "label": "completed_group_2", "mean": 0.09476494789123535, "median": 0.09293568134307861, "p05": 0.06641139835119247, "p95": 0.13004404306411743 }, { "label": "completed_group_3", "mean": 0.1030701994895935, "median": 0.099480040371418, "p05": 0.06904599815607071, "p95": 0.14615577459335327 }, { "label": "completed_group_4", "mean": 0.08048167824745178, "median": 0.07773597538471222, "p05": 0.052633628249168396, "p95": 0.12012942135334015 }, { "label": "completed_group_5", "mean": 0.12287819385528564, "median": 0.11439820379018784, "p05": 0.07120100408792496, "p95": 0.2025740146636963 }, { "label": "completed_group_6", "mean": 0.22555868327617645, "median": 0.22237449884414673, "p05": 0.12643620371818542, "p95": 0.3325231373310089 }, { "label": "current_group_partial", "mean": 0.19202858209609985, "median": 0.17407171428203583, "p05": 0.06696207821369171, "p95": 0.3700798749923706 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0024678707122803, "group": 7, "latest_source_mass": 0.19086343050003052, "layer": 28, "max_source_mass": 0.19086343050003052, "mixer_index": 54, "normalized_entropy": 0.9629834888720297, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10720443725585938, "median": 0.0965266302227974, "p05": 0.044251296669244766, "p95": 0.18923941254615784 }, { "label": "completed_group_1", "mean": 0.09982497245073318, "median": 0.09860500693321228, "p05": 0.05300002545118332, "p95": 0.15633785724639893 }, { "label": "completed_group_2", "mean": 0.10896851122379303, "median": 0.1086052879691124, "p05": 0.07999510318040848, "p95": 0.13937662541866302 }, { "label": "completed_group_3", "mean": 0.1345594972372055, "median": 0.1320592314004898, "p05": 0.10167843103408813, "p95": 0.17577221989631653 }, { "label": "completed_group_4", "mean": 0.09260416775941849, "median": 0.09119044244289398, "p05": 0.06200484186410904, "p95": 0.1281491219997406 }, { "label": "completed_group_5", "mean": 0.1016254797577858, "median": 0.09927940368652344, "p05": 0.06724471598863602, "p95": 0.14429867267608643 }, { "label": "completed_group_6", "mean": 0.1643495112657547, "median": 0.16063156723976135, "p05": 0.11097142100334167, "p95": 0.2319793701171875 }, { "label": "current_group_partial", "mean": 0.19086343050003052, "median": 0.17437744140625, "p05": 0.08801081776618958, "p95": 0.33255329728126526 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.9389225244522095, "group": 7, "latest_source_mass": 0.1535017192363739, "layer": 28, "max_source_mass": 0.2661469578742981, "mixer_index": 55, "normalized_entropy": 0.932424636898371, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08632777631282806, "median": 0.06584042310714722, "p05": 0.04890177398920059, "p95": 0.24412202835083008 }, { "label": "completed_group_1", "mean": 0.09041796624660492, "median": 0.08850662410259247, "p05": 0.06795615702867508, "p95": 0.1180017739534378 }, { "label": "completed_group_2", "mean": 0.09941554069519043, "median": 0.0980331301689148, "p05": 0.06698913872241974, "p95": 0.13627989590168 }, { "label": "completed_group_3", "mean": 0.10805265605449677, "median": 0.10531660169363022, "p05": 0.07596777379512787, "p95": 0.14889071881771088 }, { "label": "completed_group_4", "mean": 0.07865813374519348, "median": 0.07749675214290619, "p05": 0.05567489564418793, "p95": 0.10581911355257034 }, { "label": "completed_group_5", "mean": 0.11747924983501434, "median": 0.11117055267095566, "p05": 0.0703330934047699, "p95": 0.18614599108695984 }, { "label": "completed_group_6", "mean": 0.2661469578742981, "median": 0.2658160924911499, "p05": 0.15578627586364746, "p95": 0.3771526515483856 }, { "label": "current_group_partial", "mean": 0.1535017192363739, "median": 0.13699908554553986, "p05": 0.0626269057393074, "p95": 0.29617446660995483 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.008256673812866, "group": 8, "latest_source_mass": 0.11735987663269043, "layer": 29, "max_source_mass": 0.20522482693195343, "mixer_index": 56, "normalized_entropy": 0.9657673147139957, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1016545444726944, "median": 0.07978972047567368, "p05": 0.04056411236524582, "p95": 0.2331526130437851 }, { "label": "completed_group_1", "mean": 0.10481879115104675, "median": 0.1020151898264885, "p05": 0.07684041559696198, "p95": 0.1442326009273529 }, { "label": "completed_group_2", "mean": 0.10823093354701996, "median": 0.10697583854198456, "p05": 0.08568178862333298, "p95": 0.13496363162994385 }, { "label": "completed_group_3", "mean": 0.1270347535610199, "median": 0.12582050263881683, "p05": 0.1004788875579834, "p95": 0.15752868354320526 }, { "label": "completed_group_4", "mean": 0.0919865146279335, "median": 0.09061764925718307, "p05": 0.06585606187582016, "p95": 0.1224639043211937 }, { "label": "completed_group_5", "mean": 0.14368976652622223, "median": 0.14165253937244415, "p05": 0.09602011740207672, "p95": 0.20366834104061127 }, { "label": "completed_group_6", "mean": 0.20522482693195343, "median": 0.2038867175579071, "p05": 0.14235222339630127, "p95": 0.27111250162124634 }, { "label": "completed_group_7", "mean": 0.11735987663269043, "median": 0.10386677086353302, "p05": 0.057839710265398026, "p95": 0.21596315503120422 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.094825267791748, "group": 8, "latest_source_mass": 0.16477900743484497, "layer": 29, "max_source_mass": 0.19253811240196228, "mixer_index": 57, "normalized_entropy": 0.9533960658365591, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08519881963729858, "median": 0.07928745448589325, "p05": 0.0396789014339447, "p95": 0.16862331330776215 }, { "label": "completed_group_1", "mean": 0.09903685003519058, "median": 0.09788446873426437, "p05": 0.0575668141245842, "p95": 0.14432772994041443 }, { "label": "completed_group_2", "mean": 0.08900875598192215, "median": 0.08856695145368576, "p05": 0.0673285573720932, "p95": 0.11194870620965958 }, { "label": "completed_group_3", "mean": 0.09697237610816956, "median": 0.09451373666524887, "p05": 0.0689435601234436, "p95": 0.13275021314620972 }, { "label": "completed_group_4", "mean": 0.071251779794693, "median": 0.0703696608543396, "p05": 0.049036141484975815, "p95": 0.09625756740570068 }, { "label": "completed_group_5", "mean": 0.09201780706644058, "median": 0.08780405670404434, "p05": 0.04908040910959244, "p95": 0.14967313408851624 }, { "label": "completed_group_6", "mean": 0.19253811240196228, "median": 0.18946462869644165, "p05": 0.12099648267030716, "p95": 0.27126166224479675 }, { "label": "completed_group_7", "mean": 0.1091964989900589, "median": 0.09459474682807922, "p05": 0.04855653643608093, "p95": 0.2098814696073532 }, { "label": "current_group_partial", "mean": 0.16477900743484497, "median": 0.16038388013839722, "p05": 0.08161723613739014, "p95": 0.2644755244255066 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.140371322631836, "group": 8, "latest_source_mass": 0.10623672604560852, "layer": 30, "max_source_mass": 0.14476820826530457, "mixer_index": 58, "normalized_entropy": 0.9741249687033317, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10001853853464127, "median": 0.09240126609802246, "p05": 0.025385506451129913, "p95": 0.22696438431739807 }, { "label": "completed_group_1", "mean": 0.10924500226974487, "median": 0.10823708027601242, "p05": 0.08299213647842407, "p95": 0.13906806707382202 }, { "label": "completed_group_2", "mean": 0.10232415795326233, "median": 0.09996328502893448, "p05": 0.07388727366924286, "p95": 0.1385582983493805 }, { "label": "completed_group_3", "mean": 0.11669868975877762, "median": 0.11613668501377106, "p05": 0.08647200465202332, "p95": 0.1492128223180771 }, { "label": "completed_group_4", "mean": 0.10123330354690552, "median": 0.10036572813987732, "p05": 0.07442998141050339, "p95": 0.13040892779827118 }, { "label": "completed_group_5", "mean": 0.1287832111120224, "median": 0.1245323196053505, "p05": 0.08930385112762451, "p95": 0.18039579689502716 }, { "label": "completed_group_6", "mean": 0.14476820826530457, "median": 0.14273962378501892, "p05": 0.09932437539100647, "p95": 0.19615304470062256 }, { "label": "completed_group_7", "mean": 0.09069214761257172, "median": 0.08483204245567322, "p05": 0.04427404701709747, "p95": 0.15117761492729187 }, { "label": "current_group_partial", "mean": 0.10623672604560852, "median": 0.08798803389072418, "p05": 0.0472906231880188, "p95": 0.21512800455093384 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.117004632949829, "group": 8, "latest_source_mass": 0.13358455896377563, "layer": 30, "max_source_mass": 0.17978665232658386, "mixer_index": 59, "normalized_entropy": 0.963490329930842, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09184888750314713, "median": 0.08941377699375153, "p05": 0.05124857276678085, "p95": 0.14178073406219482 }, { "label": "completed_group_1", "mean": 0.09620077908039093, "median": 0.0951167494058609, "p05": 0.06255991011857986, "p95": 0.1391918957233429 }, { "label": "completed_group_2", "mean": 0.09620209783315659, "median": 0.09759913384914398, "p05": 0.06909230351448059, "p95": 0.11898315697908401 }, { "label": "completed_group_3", "mean": 0.10222947597503662, "median": 0.10126610100269318, "p05": 0.07587265968322754, "p95": 0.1305830329656601 }, { "label": "completed_group_4", "mean": 0.06924222409725189, "median": 0.06880369037389755, "p05": 0.05338519439101219, "p95": 0.08691392093896866 }, { "label": "completed_group_5", "mean": 0.0821654349565506, "median": 0.07825584709644318, "p05": 0.053967058658599854, "p95": 0.12216144800186157 }, { "label": "completed_group_6", "mean": 0.17978665232658386, "median": 0.17625530064105988, "p05": 0.11167110502719879, "p95": 0.2575375437736511 }, { "label": "completed_group_7", "mean": 0.14873988926410675, "median": 0.1422305703163147, "p05": 0.08190920948982239, "p95": 0.23667879402637482 }, { "label": "current_group_partial", "mean": 0.13358455896377563, "median": 0.12330813705921173, "p05": 0.06055498123168945, "p95": 0.23775091767311096 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1423683166503906, "group": 8, "latest_source_mass": 0.11159604042768478, "layer": 31, "max_source_mass": 0.1649385243654251, "mixer_index": 60, "normalized_entropy": 0.9750338398488454, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09596475213766098, "median": 0.0976155698299408, "p05": 0.04209773987531662, "p95": 0.15498679876327515 }, { "label": "completed_group_1", "mean": 0.07681103050708771, "median": 0.07558202743530273, "p05": 0.04380175098776817, "p95": 0.11200148612260818 }, { "label": "completed_group_2", "mean": 0.11225531995296478, "median": 0.11153945326805115, "p05": 0.08037804812192917, "p95": 0.1473693549633026 }, { "label": "completed_group_3", "mean": 0.11592596024274826, "median": 0.115887850522995, "p05": 0.08687994629144669, "p95": 0.14547690749168396 }, { "label": "completed_group_4", "mean": 0.0810609757900238, "median": 0.08056386560201645, "p05": 0.06259842216968536, "p95": 0.10083132982254028 }, { "label": "completed_group_5", "mean": 0.1177809089422226, "median": 0.11515317857265472, "p05": 0.08443494141101837, "p95": 0.15806551277637482 }, { "label": "completed_group_6", "mean": 0.1649385243654251, "median": 0.16307872533798218, "p05": 0.12056200951337814, "p95": 0.21692173182964325 }, { "label": "completed_group_7", "mean": 0.12366649508476257, "median": 0.12186841666698456, "p05": 0.07334212958812714, "p95": 0.18257394433021545 }, { "label": "current_group_partial", "mean": 0.11159604042768478, "median": 0.10134477913379669, "p05": 0.05939295515418053, "p95": 0.20170332491397858 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.093705415725708, "group": 8, "latest_source_mass": 0.09642469882965088, "layer": 31, "max_source_mass": 0.21980023384094238, "mixer_index": 61, "normalized_entropy": 0.9528863991972947, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08838656544685364, "median": 0.07248079776763916, "p05": 0.05256194248795509, "p95": 0.199167400598526 }, { "label": "completed_group_1", "mean": 0.0955723449587822, "median": 0.09313549846410751, "p05": 0.07088664174079895, "p95": 0.12816612422466278 }, { "label": "completed_group_2", "mean": 0.09992480278015137, "median": 0.10067024827003479, "p05": 0.06702706217765808, "p95": 0.13047263026237488 }, { "label": "completed_group_3", "mean": 0.10721968114376068, "median": 0.10629363358020782, "p05": 0.07515021413564682, "p95": 0.14141342043876648 }, { "label": "completed_group_4", "mean": 0.07486072182655334, "median": 0.07471001148223877, "p05": 0.055637944489717484, "p95": 0.09505966305732727 }, { "label": "completed_group_5", "mean": 0.09290309250354767, "median": 0.09035329520702362, "p05": 0.057027287781238556, "p95": 0.1383095532655716 }, { "label": "completed_group_6", "mean": 0.21980023384094238, "median": 0.21675530076026917, "p05": 0.13081300258636475, "p95": 0.3184159994125366 }, { "label": "completed_group_7", "mean": 0.12490786612033844, "median": 0.11867299675941467, "p05": 0.06415548175573349, "p95": 0.21230830252170563 }, { "label": "current_group_partial", "mean": 0.09642469882965088, "median": 0.08079533278942108, "p05": 0.036796458065509796, "p95": 0.20610423386096954 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1627297401428223, "group": 8, "latest_source_mass": 0.09715649485588074, "layer": 32, "max_source_mass": 0.13960200548171997, "mixer_index": 62, "normalized_entropy": 0.9843007230352316, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.0994885265827179, "median": 0.10293936729431152, "p05": 0.037643272429704666, "p95": 0.1762332022190094 }, { "label": "completed_group_1", "mean": 0.09216944873332977, "median": 0.09255991876125336, "p05": 0.07000453770160675, "p95": 0.11256910115480423 }, { "label": "completed_group_2", "mean": 0.11588040739297867, "median": 0.11537714302539825, "p05": 0.09686640650033951, "p95": 0.1361614316701889 }, { "label": "completed_group_3", "mean": 0.12231601774692535, "median": 0.12174015492200851, "p05": 0.10175298154354095, "p95": 0.14450547099113464 }, { "label": "completed_group_4", "mean": 0.09873614460229874, "median": 0.09799051284790039, "p05": 0.07405688613653183, "p95": 0.12650127708911896 }, { "label": "completed_group_5", "mean": 0.13436099886894226, "median": 0.13138467073440552, "p05": 0.1039634495973587, "p95": 0.17342084646224976 }, { "label": "completed_group_6", "mean": 0.13960200548171997, "median": 0.1386084109544754, "p05": 0.10816053301095963, "p95": 0.17382028698921204 }, { "label": "completed_group_7", "mean": 0.1002899557352066, "median": 0.10035285353660583, "p05": 0.06299241632223129, "p95": 0.14122843742370605 }, { "label": "current_group_partial", "mean": 0.09715649485588074, "median": 0.08847378194332123, "p05": 0.05721396207809448, "p95": 0.1772533655166626 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1300058364868164, "group": 8, "latest_source_mass": 0.07372864335775375, "layer": 32, "max_source_mass": 0.1931532621383667, "mixer_index": 63, "normalized_entropy": 0.9694074326572047, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.0899420753121376, "median": 0.07823143899440765, "p05": 0.051866769790649414, "p95": 0.19706517457962036 }, { "label": "completed_group_1", "mean": 0.1065598577260971, "median": 0.10517844557762146, "p05": 0.08068235963582993, "p95": 0.13615867495536804 }, { "label": "completed_group_2", "mean": 0.11013364791870117, "median": 0.11150452494621277, "p05": 0.07762482017278671, "p95": 0.1385573297739029 }, { "label": "completed_group_3", "mean": 0.12377548217773438, "median": 0.12257742881774902, "p05": 0.0912313312292099, "p95": 0.15917643904685974 }, { "label": "completed_group_4", "mean": 0.08726659417152405, "median": 0.08603408932685852, "p05": 0.06540867686271667, "p95": 0.11203364282846451 }, { "label": "completed_group_5", "mean": 0.11411209404468536, "median": 0.11123606562614441, "p05": 0.07641662657260895, "p95": 0.16032427549362183 }, { "label": "completed_group_6", "mean": 0.1931532621383667, "median": 0.19350019097328186, "p05": 0.12057775259017944, "p95": 0.26759904623031616 }, { "label": "completed_group_7", "mean": 0.10132836550474167, "median": 0.09569840133190155, "p05": 0.06355049461126328, "p95": 0.15627986192703247 }, { "label": "current_group_partial", "mean": 0.07372864335775375, "median": 0.0687510147690773, "p05": 0.04834525287151337, "p95": 0.1152075007557869 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.9258099794387817, "group": null, "latest_source_mass": 0.1844884157180786, "layer": null, "max_source_mass": 0.2706507444381714, "mixer_index": null, "normalized_entropy": 0.8764738931573011, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.11242504417896271, "median": 0.11513695120811462, "p05": 0.04474085196852684, "p95": 0.16795574128627777 }, { "label": "completed_group_1", "mean": 0.0684729665517807, "median": 0.06829436123371124, "p05": 0.046843402087688446, "p95": 0.09099384397268295 }, { "label": "completed_group_2", "mean": 0.07014584541320801, "median": 0.06858030706644058, "p05": 0.05111496150493622, "p95": 0.09375781565904617 }, { "label": "completed_group_3", "mean": 0.059657223522663116, "median": 0.057276882231235504, "p05": 0.037933796644210815, "p95": 0.08897658437490463 }, { "label": "completed_group_4", "mean": 0.05741159990429878, "median": 0.05489874631166458, "p05": 0.03844546526670456, "p95": 0.08331313729286194 }, { "label": "completed_group_5", "mean": 0.05127572640776634, "median": 0.04766879230737686, "p05": 0.026877565309405327, "p95": 0.08776431530714035 }, { "label": "completed_group_6", "mean": 0.12547241151332855, "median": 0.10524329543113708, "p05": 0.04063908010721207, "p95": 0.28077831864356995 }, { "label": "completed_group_7", "mean": 0.2706507444381714, "median": 0.2597617506980896, "p05": 0.09951092302799225, "p95": 0.4855192005634308 }, { "label": "completed_group_8", "mean": 0.1844884157180786, "median": 0.16994649171829224, "p05": 0.03097841702401638, "p95": 0.39343374967575073 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "e4c617a86986c3d60218f9eb527fd075b82f9473140115b430b5480c040907aa", "optimizer_state_before": "e4c617a86986c3d60218f9eb527fd075b82f9473140115b430b5480c040907aa", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "9fd6aaea3d231a1caa717e4fc44c63b6ac0b7fcac0845e349ac60b57e28fd322", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.05045261234045029, 0.09517527371644974, 0.08140470087528229, 0.035610511898994446, 0.1042611226439476, 0.050615254789590836, 0.04264022409915924, 0.03225839138031006, 0.040291935205459595, 0.036216676235198975, 0.02871432714164257, 0.03194425627589226, 0.05663999170064926, 0.03894488140940666, 0.03595346212387085, 0.03414257988333702, 0.03680795058608055, 0.022694969549775124, 0.022243596613407135, 0.02950950525701046, 0.035728659480810165, 0.024367405101656914, 0.0301151555031538, 0.022455131635069847, 0.025910304859280586, 0.025968369096517563, 0.01597527042031288, 0.015805471688508987, 0.013562481850385666, 0.02184087596833706, 0.012465676292777061, 0.01876053772866726 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.710775745777937e-05, "first_to_last_ratio": 1.9385534866869027, "imbalance_abs_log_ratio": 0.6619420695392257, "last_quartile_mean": 8.82501183241402e-06, "mean": 1.510921526914899e-05, "normalized": [ 1.0736286012354828, 1.090890635760861, 1.0608178444904952, 1.0075413201072734, 1.2489868422885204, 1.2985786082191968, 1.1828591103209587, 1.094881544464814, 1.179146782695164, 1.0539743088863864, 0.9822323375070559, 0.9179899035878774, 0.7825253397308062, 0.7478217700804238, 0.7367490149494519, 0.7112256790174882, 1.3539887939888489, 1.3049151840479707, 1.2111415907817773, 1.1081141456773322, 1.8022267118607482, 1.6529820157870687, 1.4576498485204101, 1.2664807468037573, 0.9749572058937866, 0.8591646316280283, 0.7163279624937049, 0.6428460632699132, 0.42935673617045356, 0.3797395981731299, 0.34123235543755254, 0.32902676612326176 ], "peak_layer": 21, "peak_normalized": 1.8022267118607482, "population_cv": 0.35198707913936206, "rest_mean": 1.3903669713289236e-05, "spike_contrast": 1.5549248303940866, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1619161270791664e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.6221685655182227e-05, 1.648250145080965e-05, 1.602812517376151e-05, 1.5223158698063344e-05, 1.8871211068471894e-05, 1.962050373549573e-05, 1.787207293091342e-05, 1.6542800949537195e-05, 1.781598257366568e-05, 1.5924724721116945e-05, 1.4840759831713513e-05, 1.3870107068214566e-05, 1.1823343811556697e-05, 1.1299000107101165e-05, 1.1131699466204736e-05, 1.0746061889221892e-05, 2.045770816039294e-05, 1.9716244423761964e-05, 1.829939901654143e-05, 1.6742735169827938e-05, 2.7230231353314593e-05, 2.4975261112558655e-05, 2.2023945348337293e-05, 1.9135530237690546e-05, 1.4730838302057236e-05, 1.2981303370906971e-05, 1.082315338862827e-05, 9.712899554870091e-06, 6.487243354058592e-06, 5.737567335017957e-06, 5.155753115104744e-06, 4.9713362386683e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.80973648741201e-05, "first_to_last_ratio": 1.9447119155409294, "imbalance_abs_log_ratio": 0.6651138506971189, "last_quartile_mean": 3.501668516037171e-05, "mean": 6.006126523061539e-05, "normalized": [ 1.0680384089723292, 1.089396516867671, 1.0622658589778824, 1.0104708179682353, 1.2588472863894493, 1.3055677385553706, 1.1820759209316258, 1.0937244453707788, 1.1769346432718848, 1.0548790883233023, 0.9819146790647293, 0.9183563002786295, 0.7871743717726897, 0.7495710863521058, 0.7365673123693164, 0.7105643678097476, 1.3531016610201878, 1.3024588646491044, 1.2137050756314243, 1.110290286467306, 1.806416737608712, 1.6488489338485182, 1.4524952538921891, 1.2622054710738435, 0.9668831047344751, 0.8526160887571205, 0.7131274958192858, 0.6402858642446936, 0.4318749856664095, 0.3827318699453911, 0.3444766877197535, 0.332132775645838 ], "peak_layer": 21, "peak_normalized": 1.806416737608712, "population_cv": 0.35152664329216815, "rest_mean": 5.530786209554343e-05, "spike_contrast": 1.5500444043905992, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.572964216000401e-05, "top_five_layers": [ 21, 22, 23, 17, 6 ], "top_five_spike_overlap": 3 }, "values": [ 6.414773815777153e-05, 6.543053314089775e-05, 6.380103150149807e-05, 6.069015580578707e-05, 7.560796075267717e-05, 7.841405022190884e-05, 7.099697540979832e-05, 6.569047400262207e-05, 7.068818376865238e-05, 6.335737271001562e-05, 5.89750379731413e-05, 5.5157641327241436e-05, 4.727868872578256e-05, 4.502018782659434e-05, 4.4239164708415046e-05, 4.26773949584458e-05, 8.126899774651974e-05, 7.822732732165605e-05, 7.289666245924309e-05, 6.668543937848881e-05, 0.00010849567479453981, 9.903195314109325e-05, 8.723870269022882e-05, 7.580965757369995e-05, 5.807222260045819e-05, 5.120920104673132e-05, 4.2831339669646695e-05, 3.845637911581434e-05, 2.5938958060578443e-05, 2.298736035299953e-05, 2.0689705706899986e-05, 1.994831472984515e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06038942374289036, "first_to_last_ratio": 1.9447119017990018, "imbalance_abs_log_ratio": 0.6651138436308138, "last_quartile_mean": 0.03105314658023417, "mean": 0.05326293042162433, "normalized": [ 1.0680384098761833, 1.0893965216947878, 1.062265786546042, 1.0104708738830177, 1.2588473385979138, 1.3055677142391287, 1.1820758895935386, 1.09372443299358, 1.1769346283619155, 1.0548791227653083, 0.9819147305623657, 0.9183562837560109, 0.7871743890171391, 0.7495711002960661, 0.73656729318353, 0.7105643650402791, 1.3531016496377335, 1.3024589535705866, 1.2137049835246356, 1.1102903617054871, 1.8064166988298391, 1.6488489461272462, 1.4524952635466242, 1.26220537084274, 0.9668831089685697, 0.8526160811834912, 0.7131275119134806, 0.6402858670776477, 0.43187496591238844, 0.38273188596957125, 0.34447667234318236, 0.3321327984399701 ], "peak_layer": 21, "peak_normalized": 1.8064166988298391, "population_cv": 0.35152663902064574, "rest_mean": 0.049047565225649764, "spike_contrast": 1.5500443728474567, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07602590247988701, "top_five_layers": [ 21, 22, 23, 17, 6 ], "top_five_spike_overlap": 3 }, "values": [ 0.05688685551285744, 0.05802445113658905, 0.05657938867807388, 0.053820639848709106, 0.06704989820718765, 0.06953836232423782, 0.06296082586050034, 0.058254968374967575, 0.062686987221241, 0.05618595331907272, 0.052299655973911285, 0.0489143468439579, 0.041927214711904526, 0.03992435336112976, 0.03923173248767853, 0.03784674033522606, 0.0720701590180397, 0.06937278062105179, 0.06464548408985138, 0.05913731828331947, 0.09621504694223404, 0.08782252669334412, 0.07736415416002274, 0.0672287568449974, 0.051499027758836746, 0.045412831008434296, 0.03798326104879379, 0.034103501588106155, 0.023002926260232925, 0.020385421812534332, 0.018347837030887604, 0.017690366134047508 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7076921753632632e-06, "first_to_last_ratio": 1.1762812234778344, "imbalance_abs_log_ratio": 0.16235795649847604, "last_quartile_mean": 2.301908864410507e-06, "mean": 3.0470990566300316e-06, "normalized": [ 1.1138437702053292, 0.9595105662781346, 0.9024337193377198, 0.9101147747351473, 0.8469008290036252, 0.8140327082666439, 0.7830671665056109, 0.7790013618751644, 0.8724389801211856, 0.848368375017665, 0.8340257934438313, 0.8332842972958924, 0.6804006560304527, 0.6823355453447383, 0.6859143070691168, 0.7155834804515504, 1.343985609139468, 1.324647908954466, 1.3004430629764219, 1.269459612482823, 2.009601561760871, 1.969745677435912, 1.8244349675950344, 1.6528833381228312, 1.244173825255001, 1.0943835444416905, 0.9361801141676105, 0.8608013633619623, 0.5518818827305038, 0.48726512359119484, 0.4398138473062473, 0.429042229696154 ], "peak_layer": 21, "peak_normalized": 2.009601561760871, "population_cv": 0.4073949027659483, "rest_mean": 2.629438902793616e-06, "spike_contrast": 2.0165761911079723, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.302463887346676e-06, "top_five_layers": [ 21, 22, 23, 24, 17 ], "top_five_spike_overlap": 4 }, "values": [ 3.3939923014258966e-06, 2.923723741332651e-06, 2.7498049348650966e-06, 2.773209871520521e-06, 2.580590717116138e-06, 2.4804382974252803e-06, 2.3860832243371988e-06, 2.3736943148833234e-06, 2.6584079932945315e-06, 2.58506247519108e-06, 2.541359208407812e-06, 2.5390997961949324e-06, 2.073248197120847e-06, 2.07914399652509e-06, 2.0900488379993476e-06, 2.180453748223954e-06, 4.095257281733211e-06, 4.0363333937420975e-06, 3.962578830396524e-06, 3.868169187626336e-06, 6.123455023043789e-06, 6.00201019551605e-06, 5.559234068641672e-06, 5.036499260313576e-06, 3.7911208892182913e-06, 3.334695065859705e-06, 2.852633542715921e-06, 2.6229470222460805e-06, 1.6816387642393238e-06, 1.4847450984234456e-06, 1.3401563592196908e-06, 1.3073341733615962e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.342586018741713e-05, "first_to_last_ratio": 2.2394319733492734, "imbalance_abs_log_ratio": 0.806222250383043, "last_quartile_mean": 2.385688014783227e-05, "mean": 4.457031414517587e-05, "normalized": [ 1.1494422787070995, 1.1718836421223593, 1.1428090747145467, 1.086444308176937, 1.3229548508092461, 1.3578460621138582, 1.2296177221512083, 1.1284987855754436, 1.227111884992109, 1.0975534107511307, 1.0166764145258342, 0.9402072429997458, 0.8277192334474507, 0.78847643593883, 0.7746430720935581, 0.7477481748973609, 1.3079701078445523, 1.262663592527712, 1.1708787769345523, 1.070669614331925, 1.7921884489124142, 1.5923472008639945, 1.3491773347638585, 1.1623611344243405, 0.9084315137318004, 0.7836848181998147, 0.6646284136029903, 0.6021622600764145, 0.37448601778033414, 0.3346427580221723, 0.3104097006323165, 0.30366571333408976 ], "peak_layer": 21, "peak_normalized": 1.7921884489124142, "population_cv": 0.35707967857039985, "rest_mean": 4.1591522184952975e-05, "spike_contrast": 1.458369098891111, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.0655790730379525e-05, "top_five_layers": [ 21, 22, 6, 23, 5 ], "top_five_spike_overlap": 3 }, "values": [ 5.1231003453722224e-05, 5.2231222070986405e-05, 5.093535946798511e-05, 4.842316411668435e-05, 5.896451330045238e-05, 6.051962554920465e-05, 5.480444815475494e-05, 5.029754538554698e-05, 5.469276220537722e-05, 4.8918300308287144e-05, 4.531358717940748e-05, 4.190533218206838e-05, 3.689170625875704e-05, 3.514264244586229e-05, 3.4526085073594004e-05, 3.3327371056657284e-05, 5.8296638599131256e-05, 5.6277312978636473e-05, 5.21864349138923e-05, 4.772008105646819e-05, 7.987840217538178e-05, 7.09714149706997e-05, 6.013325764797628e-05, 5.180680091143586e-05, 4.0489077946403995e-05, 3.492907853797078e-05, 2.962269718409516e-05, 2.683856109797489e-05, 1.669095945544541e-05, 1.4915132851456292e-05, 1.3835057870892342e-05, 1.3534476238419302e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.062916423208662e-05, "first_to_last_ratio": 2.3115003864920087, "imbalance_abs_log_ratio": 0.837896831650113, "last_quartile_mean": 1.7576966229171376e-05, "mean": 3.283533709463882e-05, "normalized": [ 1.212911398116158, 1.2408980091883677, 1.2100379395586904, 1.1493036508582075, 1.3478891639463513, 1.376547190436269, 1.2400156408077365, 1.1212846877559697, 1.2545199865442007, 1.1255636869056884, 1.0416509414039974, 0.9496446486640949, 0.863811056776793, 0.8232971746250972, 0.814426624693093, 0.7852693022328948, 1.2280636708101726, 1.1957979497936206, 1.0979825841550646, 1.0020961529096264, 1.784118618704249, 1.5269728290156659, 1.2477064495899464, 1.0777394093111723, 0.8973943474546292, 0.7708740338031522, 0.6829522402516178, 0.6279739538251877, 0.35258776830231575, 0.32638760973755176, 0.31414801689768274, 0.3101332629247351 ], "peak_layer": 21, "peak_normalized": 1.784118618704249, "population_cv": 0.3491548585302889, "rest_mean": 3.096988661531618e-05, "spike_contrast": 1.385499734498883, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2908769682981077e-05, "top_five_layers": [ 21, 22, 6, 5, 9 ], "top_five_spike_overlap": 2 }, "values": [ 3.982635462307371e-05, 4.074530443176627e-05, 3.973200364271179e-05, 3.7737772800028324e-05, 4.425839506438933e-05, 4.519939102465287e-05, 4.071633156854659e-05, 3.6817760701524094e-05, 4.1192586650140584e-05, 3.695826308103278e-05, 3.420295979594812e-05, 3.1181902159005404e-05, 2.836352723534219e-05, 2.7033240257878788e-05, 2.6741972760646604e-05, 2.578458224888891e-05, 4.0323884604731575e-05, 3.926442877855152e-05, 3.605262827477418e-05, 3.2904164982028306e-05, 5.858213626197539e-05, 5.013866757508367e-05, 4.0968861867440864e-05, 3.538793680490926e-05, 2.946624590549618e-05, 2.53119087574305e-05, 2.2424967028200626e-05, 2.061973646050319e-05, 1.1577338227652945e-05, 1.0717047189245932e-05, 1.0315156032447703e-05, 1.0183330232393928e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013562339336203877, "first_to_last_ratio": 1.8937266056606614, "imbalance_abs_log_ratio": 0.6385466367527224, "last_quartile_mean": 7.161719804571476e-05, "mean": 0.00012335994665590988, "normalized": [ 1.019000781731899, 1.0322486217644518, 1.0117188048559849, 0.9776595701209428, 1.2173099941502679, 1.2992497220260693, 1.1640291527018616, 1.0740785542805755, 1.1554579576814683, 1.0322293937877338, 0.9611972370481242, 0.9118663882402063, 0.7737203353417383, 0.7258140672751332, 0.7097483849490263, 0.6892843914209803, 1.412411909274259, 1.3466569474208636, 1.2405204033466979, 1.1273987953867963, 1.8525995137972373, 1.7348580010509511, 1.5314488922065341, 1.355054500835559, 0.9639698405130199, 0.8534217733756123, 0.7099903271591395, 0.6468017103823497, 0.4317554879354095, 0.37917174911531354, 0.33471259721834695, 0.32461419360544697 ], "peak_layer": 21, "peak_normalized": 1.8525995137972373, "population_cv": 0.370445431994308, "rest_mean": 0.00011222131676428641, "spike_contrast": 1.6352378795922027, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018350854807067663, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012570388207677752, 0.00012733813491649926, 0.0001248055777978152, 0.0001206040324177593, 0.000150167295942083, 0.00016027537640184164, 0.00014359457418322563, 0.0001324982731603086, 0.0001425372320227325, 0.00012733576295431703, 0.00011857323988806456, 0.00011248778901062906, 9.544609929434955e-05, 8.953638462116942e-05, 8.755452290643007e-05, 8.503008575644344e-05, 0.00017423505778424442, 0.00016612352919764817, 0.00015303053078241646, 0.00013907585525885224, 0.00022853657719679177, 0.0002140119904652238, 0.00018891945364885032, 0.00016715945093892515, 0.0001189152681035921, 0.00010527806443860754, 8.758436888456345e-05, 7.978942448971793e-05, 5.326133396010846e-05, 4.6774606744293123e-05, 4.1290128137916327e-05, 4.004438960691914e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "884b74f81575660d3a5ef373d83980c2b6a40a5e790421e3f7efc3bf3cdbb781", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0860176682472229, 0.031032679602503777, 0.011454286053776741, 0.03601690009236336, 0.01171922218054533, 0.008982792496681213, 0.01206296682357788, 0.026524480432271957, 0.024220434948801994, 0.026388581842184067, 0.03031591884791851, 0.032942064106464386, 0.029961559921503067, 0.02937687374651432, 0.031781863421201706, 0.035546254366636276, 0.030760737136006355, 0.03042135201394558, 0.03508196026086807, 0.04756028205156326, 0.0295625701546669, 0.03350908309221268, 0.038334235548973083, 0.043223608285188675, 0.04338310658931732, 0.046161677688360214, 0.050398919731378555, 0.062427736818790436, 0.0627857968211174, 0.08928166329860687, 0.09919292479753494, 0.13455425202846527 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6889636526684626e-05, "first_to_last_ratio": 2.126000037178537, "imbalance_abs_log_ratio": 0.7542422974073087, "last_quartile_mean": 7.944325602693425e-06, "mean": 1.4253426712684814e-05, "normalized": [ 1.115336759287406, 1.1464967880052945, 1.1249731711222886, 1.0622402704566627, 1.317199232206508, 1.3595432924960245, 1.2293188827089512, 1.1245132369645223, 1.155858821804884, 1.0614255591183779, 0.9912125389310478, 0.9372185786895139, 0.8026693967866093, 0.7700318104442434, 0.7622598930724469, 0.7357685661471873, 1.3571059222187842, 1.2941689608655147, 1.1908997045010374, 1.0892684211984158, 1.7975523481237365, 1.5821682703899984, 1.3498998616881461, 1.1839696810356735, 0.918100044807967, 0.760612540110315, 0.6815993230072928, 0.6358967479480955, 0.40490244236710804, 0.3615503937465385, 0.34857773642902556, 0.3476608033203826 ], "peak_layer": 21, "peak_normalized": 1.7975523481237365, "population_cv": 0.3491891089141571, "rest_mean": 1.3286468856676947e-05, "spike_contrast": 1.4657769001837064, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.947499913512729e-05, "top_five_layers": [ 21, 22, 6, 17, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.5897370758466423e-05, 1.6341507944162004e-05, 1.603472264832817e-05, 1.5140563846216537e-05, 1.877460272226017e-05, 1.93781506823143e-05, 1.7522006601211615e-05, 1.602816701051779e-05, 1.6474949006806128e-05, 1.51289514178643e-05, 1.4128175280347932e-05, 1.335857632511761e-05, 1.1440789421612862e-05, 1.0975591976603027e-05, 1.0864815521927085e-05, 1.0487223335076123e-05, 1.9343409803695977e-05, 1.8446342437528074e-05, 1.697440166026354e-05, 1.5525807611993514e-05, 2.5621280656196177e-05, 2.2551319489139132e-05, 1.9240698748035356e-05, 1.6875625078682788e-05, 1.3086071703583002e-05, 1.0841335097211413e-05, 9.715125997900032e-06, 9.063707693712786e-06, 5.771247288066661e-06, 5.153332040208625e-06, 4.9684272198646795e-06, 4.9553577810002025e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.725307594024343e-05, "first_to_last_ratio": 2.1308782233368584, "imbalance_abs_log_ratio": 0.7565342061700902, "last_quartile_mean": 3.156120101266424e-05, "mean": 5.6670199910513475e-05, "normalized": [ 1.110680833704548, 1.1455034927935899, 1.1258904459559615, 1.0664054974724477, 1.3263498819954045, 1.3678166597071255, 1.2291648981942673, 1.1221482262899733, 1.1546676748054345, 1.0627574527858743, 0.9896202136771384, 0.9362005960764237, 0.8068774857623636, 0.771625357953291, 0.7613046275916601, 0.7347843884669755, 1.3577587462273788, 1.2928466997481425, 1.1909272887269375, 1.0894198852162194, 1.8057809001369873, 1.5765275443860074, 1.343499485983638, 1.1760205514263615, 0.9111321409731876, 0.7565798297150907, 0.67877158866213, 0.6338403657465395, 0.40731991397385253, 0.3648985473433193, 0.351811563377079, 0.35106721512464933 ], "peak_layer": 21, "peak_normalized": 1.8057809001369873, "population_cv": 0.34914418193489394, "rest_mean": 5.286498357977339e-05, "spike_contrast": 1.4606713720053888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.721836809650994e-05, "top_five_layers": [ 21, 22, 6, 17, 23 ], "top_five_spike_overlap": 3 }, "values": [ 6.29425048828125e-05, 6.491591193480417e-05, 6.380443664966151e-05, 6.043341272743419e-05, 7.516451296396554e-05, 7.751444354653358e-05, 6.965702050365508e-05, 6.359236431308091e-05, 6.543524796143174e-05, 6.022667730576359e-05, 5.608197534456849e-05, 5.305467493599281e-05, 4.572590842144564e-05, 4.372816329123452e-05, 4.314328543841839e-05, 4.164037818554789e-05, 7.694445957895368e-05, 7.326588092837483e-05, 6.749008753104135e-05, 6.17376426816918e-05, 0.00010233396460535005, 8.934213110478595e-05, 7.613638445036486e-05, 6.66453197482042e-05, 5.1634040573844686e-05, 4.287553019821644e-05, 3.8466121623059735e-05, 3.591986023820937e-05, 2.3082900952431373e-05, 2.067887362500187e-05, 1.9937231627409346e-05, 1.9895049263141118e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.059640700463205576, "first_to_last_ratio": 2.1308782476695067, "imbalance_abs_log_ratio": 0.7565342175891604, "last_quartile_mean": 0.02798878843896091, "mean": 0.05025569983990863, "normalized": [ 1.1106807805227414, 1.1455035131670817, 1.1258903978802657, 1.0664054823966183, 1.3263498841814558, 1.3678166690587785, 1.229165007333235, 1.1221482585659923, 1.1546676515729808, 1.0627574840194747, 0.9896202794839417, 0.9362006347867332, 0.8068774504100944, 0.7716253757910643, 0.7613046381394664, 0.7347843948827595, 1.3577587106372004, 1.292846681318099, 1.1909273307261892, 1.0894199023292641, 1.8057809158157074, 1.576527571436041, 1.3434993978576328, 1.1760204469019233, 0.9111320892566023, 0.7565798749963789, 0.6787715750645948, 0.6338403295741707, 0.40731992625007324, 0.36489853744490924, 0.35181157583949707, 0.3510672323590328 ], "peak_layer": 21, "peak_normalized": 1.8057809158157074, "population_cv": 0.34914418003675807, "rest_mean": 0.04688119633054292, "spike_contrast": 1.46067131708135, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.06847801879048347, "top_five_layers": [ 21, 22, 6, 17, 23 ], "top_five_spike_overlap": 3 }, "values": [ 0.055818039923906326, 0.057568080723285675, 0.056582409888505936, 0.05359295383095741, 0.06665664166212082, 0.06874058395624161, 0.06177254766225815, 0.05639434605836868, 0.058028630912303925, 0.05340962111949921, 0.04973405972123146, 0.04704941809177399, 0.04055019095540047, 0.03877857327461243, 0.038259897381067276, 0.036927103996276855, 0.0682351142168045, 0.06497291475534439, 0.05985088646411896, 0.05474955961108208, 0.0907507836818695, 0.07922949641942978, 0.06751850247383118, 0.05910173058509827, 0.045789580792188644, 0.03802245110273361, 0.03411214053630829, 0.031854089349508286, 0.0204701479524374, 0.01833823136985302, 0.017680536955595016, 0.01764312945306301 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.6838719406896416e-06, "first_to_last_ratio": 1.2889488096889063, "imbalance_abs_log_ratio": 0.25382700997133584, "last_quartile_mean": 2.082217633869732e-06, "mean": 2.8460147731834695e-06, "normalized": [ 1.1585408375114765, 1.0223791463270808, 0.9675822950955996, 0.9815034677189116, 0.9061206974504489, 0.8633297690787293, 0.8261227289030565, 0.818645960689999, 0.8917059523481307, 0.8654323654695489, 0.843182297073924, 0.8552717070234496, 0.7079172332660071, 0.7114323992588514, 0.7214302379069232, 0.7529703821484969, 1.351183821841786, 1.2964760501367036, 1.2560631856060487, 1.2250133382734523, 2.006580964722333, 1.883398280372172, 1.6561147848218503, 1.578596263002518, 1.1809705030150337, 0.9928005092892233, 0.9060768367688408, 0.8769753144678925, 0.5241234809909122, 0.4617266282105258, 0.45178683106552914, 0.4585457301445449 ], "peak_layer": 21, "peak_normalized": 2.006580964722333, "population_cv": 0.37720922393237755, "rest_mean": 2.4975718302071242e-06, "spike_contrast": 1.8928811608448006, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.727606665255735e-06, "top_five_layers": [ 21, 22, 23, 24, 17 ], "top_five_spike_overlap": 4 }, "values": [ 3.2972243388940115e-06, 2.909706154241576e-06, 2.7537535061128438e-06, 2.793373369058827e-06, 2.5788328912312863e-06, 2.457049276927137e-06, 2.351157490920741e-06, 2.329878498130711e-06, 2.537808313718415e-06, 2.463033297317452e-06, 2.3997092739591608e-06, 2.4341159132745815e-06, 2.0147429040662246e-06, 2.0247471184120513e-06, 2.0532011149043683e-06, 2.1429648313642247e-06, 3.845489118248224e-06, 3.6897899917676114e-06, 3.574774382286705e-06, 3.486406058073044e-06, 5.710759069188498e-06, 5.3601793297275435e-06, 4.713327143690549e-06, 4.492708285397384e-06, 3.361059498274699e-06, 2.8255249162612017e-06, 2.578708063083468e-06, 2.495884700692841e-06, 1.4916631698724814e-06, 1.3140808050593478e-06, 1.2857919955422403e-06, 1.3050279221715755e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.28848186149844e-05, "first_to_last_ratio": 2.4318675102825495, "imbalance_abs_log_ratio": 0.8886594849400852, "last_quartile_mean": 2.1746587094639835e-05, "mean": 4.2398125145837184e-05, "normalized": [ 1.1902652753716751, 1.2242846271388579, 1.2015411916375924, 1.138461083714705, 1.3837339926605239, 1.4128864745548413, 1.2711291935221773, 1.1564070657346273, 1.2059250641459591, 1.1015552456548003, 1.0185066417879192, 0.9526019131078938, 0.8449353737531005, 0.8080602539480353, 0.7967414350915603, 0.7697874535477814, 1.3107593460824956, 1.2543499045498807, 1.1507767871653785, 1.055520508570848, 1.7920151228199508, 1.5166494788051519, 1.2523182952766116, 1.0874771895891975, 0.8458404468035057, 0.7039937567721195, 0.6353010461472927, 0.5963690903457104, 0.3543163425906758, 0.3268551237187333, 0.3200346200309376, 0.3206006553594606 ], "peak_layer": 21, "peak_normalized": 1.7920151228199508, "population_cv": 0.3590956605869151, "rest_mean": 4.0051623626722074e-05, "spike_contrast": 1.3749563279206762, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5069233349058776e-05, "top_five_layers": [ 21, 22, 6, 5, 17 ], "top_five_spike_overlap": 2 }, "values": [ 5.046501610195264e-05, 5.190737283555791e-05, 5.0943093810928985e-05, 4.826861550100148e-05, 5.866772698936984e-05, 5.990373756503686e-05, 5.3893494623480365e-05, 4.9029491492547095e-05, 5.112896178616211e-05, 4.6703877160325646e-05, 4.318277206039056e-05, 4.03885351261124e-05, 3.582367571652867e-05, 3.426023977226578e-05, 3.378034307388589e-05, 3.263754479121417e-05, 5.557373879128136e-05, 5.318208422977477e-05, 4.879077823716216e-05, 4.475209061638452e-05, 7.597808144055307e-05, 6.430309440474957e-05, 5.309594780555926e-05, 4.610699397744611e-05, 3.586204911698587e-05, 2.9848015401512384e-05, 2.69355732598342e-05, 2.5284931325586513e-05, 1.5022348634374794e-05, 1.3858044439984951e-05, 1.3568867871072143e-05, 1.3592866707767826e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.0377713048656005e-05, "first_to_last_ratio": 2.4735894758562855, "imbalance_abs_log_ratio": 0.905670324816002, "last_quartile_mean": 1.632353041713941e-05, "mean": 3.153969646518817e-05, "normalized": [ 1.2510101789123635, 1.284673110645709, 1.2593728815079348, 1.1934472035118944, 1.4006716541713753, 1.4203687191598957, 1.2779886928015973, 1.154218133705255, 1.2357024917885882, 1.1249430803382512, 1.0345099464756928, 0.9503377951773421, 0.8790382032687162, 0.8374457509593257, 0.832874530076665, 0.8029016908949563, 1.238017371285292, 1.1902047112336624, 1.0825047379103157, 0.9955341765531912, 1.7778096901796956, 1.4620144036626055, 1.1595163229484706, 1.0144538092925819, 0.8300456109926844, 0.7167594293866448, 0.6575434300155114, 0.6234101171676332, 0.3406962742049681, 0.32720961254604936, 0.32247976203421724, 0.32229647719091353 ], "peak_layer": 21, "peak_normalized": 1.7778096901796956, "population_cv": 0.35299231892411403, "rest_mean": 3.008672124619975e-05, "spike_contrast": 1.3090746022283983, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.938576264772564e-05, "top_five_layers": [ 21, 22, 6, 5, 2 ], "top_five_spike_overlap": 2 }, "values": [ 3.94564813177567e-05, 4.0518199966754764e-05, 3.9720238419249654e-05, 3.7640962545992807e-05, 4.4176758819958195e-05, 4.479799827095121e-05, 4.030737545690499e-05, 3.640368959167972e-05, 3.897368151228875e-05, 3.548036329448223e-05, 3.2628129702061415e-05, 2.9973365599289536e-05, 2.772459811239969e-05, 2.64127847913187e-05, 2.626860987220425e-05, 2.532327562221326e-05, 3.904669210896827e-05, 3.753869532374665e-05, 3.4141870855819434e-05, 3.1398845749208704e-05, 5.607157800113782e-05, 4.6111490519251674e-05, 3.6570792872225866e-05, 3.199556522304192e-05, 2.6179386622970924e-05, 2.2606374841416255e-05, 2.0738720195367932e-05, 1.9662165868794546e-05, 1.0745457075245213e-05, 1.0320091860194225e-05, 1.0170913810725324e-05, 1.0165133062400855e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001337056455668062, "first_to_last_ratio": 2.0677765287412297, "imbalance_abs_log_ratio": 0.7264738892699912, "last_quartile_mean": 6.466155491580139e-05, "mean": 0.00011615552659804962, "normalized": [ 1.0561568199318774, 1.0932504740214608, 1.0705240070918929, 1.0352536719647083, 1.2895031862888213, 1.3493114057481659, 1.2159685847053738, 1.0987642787605771, 1.1299239404100763, 1.0527850453988277, 0.9794864078984968, 0.9382838968850933, 0.7953293135025563, 0.743714816851873, 0.7363961718879526, 0.7207598399740149, 1.4085271780882864, 1.3361487817201736, 1.2152286835241228, 1.0939565497169452, 1.865453114985999, 1.651978845678226, 1.434528077319642, 1.2353202703920463, 0.9104842334884138, 0.7537977556803896, 0.688402255010069, 0.6517209586478964, 0.40424092626230357, 0.3552178063401815, 0.34378476083116255, 0.3457979409923738 ], "peak_layer": 21, "peak_normalized": 1.865453114985999, "population_cv": 0.3635258211024969, "rest_mean": 0.00010713082492993972, "spike_contrast": 1.5391360583069846, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016488891560584307, "top_five_layers": [ 21, 22, 23, 17, 6 ], "top_five_spike_overlap": 3 }, "values": [ 0.00012267845158930868, 0.00012698708451353014, 0.00012434727977961302, 0.00012025043542962521, 0.00014978292165324092, 0.0001567299768794328, 0.0001412414712831378, 0.00012762754340656102, 0.00013124691031407565, 0.00012228680134285241, 0.00011377275950508192, 0.0001089868601411581, 9.238189522875473e-05, 8.638658619020134e-05, 8.553648513043299e-05, 8.372023876290768e-05, 0.00016360821609850973, 0.0001552010653540492, 0.00014115552767179906, 0.0001270690991077572, 0.00021668268891517073, 0.0001918864727485925, 0.00016662836424075067, 0.00014348927652463317, 0.00010575777560006827, 8.75577752594836e-05, 7.996172644197941e-05, 7.570099114673212e-05, 4.695481766248122e-05, 4.126051135244779e-05, 3.993249993072823e-05, 4.016634193249047e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2d82179ff03320f5589762540fc93ada0edba22cabc07ab3982cf783bff4a116", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05045261234045029, 0.16565440595149994, 0.23719695210456848, 0.24811406433582306, 0.1042611226439476, 0.12873972952365875, 0.1382850706577301, 0.14404986798763275, 0.040291935205459595, 0.06678888201713562, 0.09121427685022354, 0.12122578918933868, 0.05663999170064926, 0.09547740966081619, 0.12204843014478683, 0.14941293001174927, 0.03680795058608055, 0.055156365036964417, 0.07373107224702835, 0.09760652482509613, 0.035728659480810165, 0.05559300258755684, 0.08069392293691635, 0.10087756812572479, 0.025910304859280586, 0.060516636818647385, 0.09090615063905716, 0.1273101419210434, 0.013562481850385666, 0.07113923132419586, 0.14901472628116608, 0.23766301572322845 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.7107619441958377e-05, "first_to_last_ratio": 1.9385227140552528, "imbalance_abs_log_ratio": 0.6619261953958985, "last_quartile_mean": 8.82508072663768e-06, "mean": 1.5109197732954271e-05, "normalized": [ 1.0736154005764045, 1.090864091897401, 1.0608090833759376, 1.007555611950532, 1.2489908200810547, 1.2985740959119219, 1.182842063583229, 1.0948707762633996, 1.1791532076095934, 1.0539858856617061, 0.9822340794635126, 0.9179957846165998, 0.7825363606751058, 0.7478278147754593, 0.7367534215342862, 0.7112451648667536, 1.353988198457684, 1.3048851565120956, 1.2111308371257685, 1.1081161541255893, 1.8022461396740475, 1.6529667185837162, 1.4576382974635043, 1.266481614772143, 0.974968751154223, 0.8591782697026162, 0.7163417357615641, 0.6428331451635377, 0.4293628627060633, 0.3797467205300767, 0.34123497868874664, 0.32902675673572673 ], "peak_layer": 21, "peak_normalized": 1.8022461396740475, "population_cv": 0.3519835486656121, "rest_mean": 1.3903650782325443e-05, "spike_contrast": 1.5549262279970795, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1619151266349945e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.6221467376453802e-05, 1.648208126425743e-05, 1.6027974197641015e-05, 1.522335696790833e-05, 1.8871249267249368e-05, 1.9620412786025554e-05, 1.7871794625534676e-05, 1.654261905059684e-05, 1.7816058971220627e-05, 1.592488115420565e-05, 1.484076892666053e-05, 1.3870179827790707e-05, 1.1823496606666595e-05, 1.1299078323645517e-05, 1.1131753126392141e-05, 1.0746343832579441e-05, 2.0457675418583676e-05, 1.9715767848538235e-05, 1.829921529861167e-05, 1.674274608376436e-05, 2.7230493287788704e-05, 2.497500099707395e-05, 2.2023745259502903e-05, 1.9135521142743528e-05, 1.4730995644640643e-05, 1.2981494364794344e-05, 1.0823348929989152e-05, 9.712693099572789e-06, 6.487328391813207e-06, 5.737668288929854e-06, 5.15578676640871e-06, 4.971330326952739e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.809728029111284e-05, "first_to_last_ratio": 1.944702112932975, "imbalance_abs_log_ratio": 0.6651088100365553, "last_quartile_mean": 3.501681817397184e-05, "mean": 6.006133520486401e-05, "normalized": [ 1.0680225064611175, 1.089383496884205, 1.062265590525735, 1.010490961736264, 1.2588526037326673, 1.3055648849448955, 1.1820615815511215, 1.0937235345573042, 1.1769375120625731, 1.054885733578057, 0.9819171693521337, 0.9183525046530807, 0.7871855083191073, 0.7495806312894487, 0.7365719056296469, 0.7105757147525049, 1.3531016594414176, 1.302448867275397, 1.2136850057217365, 1.1102934751860138, 1.8064271106878023, 1.6488367157824373, 1.4524898062618805, 1.2622043639721385, 0.9668821599838774, 0.8526186691123343, 0.713142716323793, 0.640285602850375, 0.431877541350555, 0.3827323628964117, 0.34447643781604553, 0.3321256653079218 ], "peak_layer": 21, "peak_normalized": 1.8064271106878023, "population_cv": 0.35152458934703573, "rest_mean": 5.530794731856548e-05, "spike_contrast": 1.5500417923139733, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.572962979087606e-05, "top_five_layers": [ 21, 22, 23, 17, 6 ], "top_five_spike_overlap": 3 }, "values": [ 6.414685776690021e-05, 6.542982737300918e-05, 6.380108970915899e-05, 6.069143637432717e-05, 7.560836820630357e-05, 7.841397018637508e-05, 7.09961968823336e-05, 6.569049583049491e-05, 7.068843842716888e-05, 6.335784564726055e-05, 5.897525625186972e-05, 5.5157477618195117e-05, 4.727941268356517e-05, 4.502081355894916e-05, 4.4239492126507685e-05, 4.267812619218603e-05, 8.126909233396873e-05, 7.822681800462306e-05, 7.28955419617705e-05, 6.668570858892053e-05, 0.00010849642421817407, 9.903133468469605e-05, 8.723847713554278e-05, 7.58096794015728e-05, 5.807223351439461e-05, 5.120941568748094e-05, 4.2832303734030575e-05, 3.84564082196448e-05, 2.59391417785082e-05, 2.298741674167104e-05, 2.0689714801847003e-05, 1.9947910914197564e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06038934923708439, "first_to_last_ratio": 1.9447021390890675, "imbalance_abs_log_ratio": 0.6651088234864776, "last_quartile_mean": 0.031053264159709215, "mean": 0.053262992412783206, "normalized": [ 1.068022549058515, 1.089383503618041, 1.062265669270574, 1.0104909600242913, 1.2588525878422883, 1.3055647959002497, 1.1820616445871126, 1.0937235097485394, 1.176937455050195, 1.0548857284650295, 0.9819172246968001, 0.91835248719328, 0.7871855727179047, 0.7495806491676233, 0.7365718913476978, 0.7105757078465339, 1.3531016135159784, 1.3024487649403136, 1.213684966508138, 1.1102934757832938, 1.8064271858558933, 1.6488366757497936, 1.4524897961943863, 1.262204461332232, 0.9668821235259304, 0.8526186558632091, 0.713142698515675, 0.6402856114586859, 0.4318775756604022, 0.3827323497583975, 0.3444764462709729, 0.3321256625320224 ], "peak_layer": 21, "peak_normalized": 1.8064271858558933, "population_cv": 0.3515245852652579, "rest_mean": 0.04904764035233745, "spike_contrast": 1.5500418163453433, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07602589353919029, "top_five_layers": [ 21, 22, 23, 17, 6 ], "top_five_spike_overlap": 3 }, "values": [ 0.05688607692718506, 0.05802382528781891, 0.05657944828271866, 0.05382177233695984, 0.0670502558350563, 0.06953828781843185, 0.06296014040708542, 0.05825498700141907, 0.0626872107386589, 0.05618637055158615, 0.052299849689006805, 0.04891420155763626, 0.04192785918712616, 0.03992490842938423, 0.03923202306032181, 0.03784738853573799, 0.07207024097442627, 0.06937231868505478, 0.06464449316263199, 0.059137552976608276, 0.09621571749448776, 0.08782197535037994, 0.07736395299434662, 0.0672287866473198, 0.05149903520941734, 0.04541302099823952, 0.03798411414027214, 0.034103527665138245, 0.023003092035651207, 0.020385470241308212, 0.01834784634411335, 0.017690006643533707 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7077437607658794e-06, "first_to_last_ratio": 1.1762941928935384, "imbalance_abs_log_ratio": 0.16236898221575077, "last_quartile_mean": 2.3019273385216366e-06, "mean": 3.047136040379428e-06, "normalized": [ 1.1138713662086555, 0.9595541384126708, 0.9023959781504268, 0.9101529768920249, 0.8469222629807022, 0.8140191718536866, 0.7830494541805194, 0.7789886983581177, 0.8724379423486327, 0.84833009613214, 0.8339719440677056, 0.8333161939360028, 0.6804733592729626, 0.6824055388257974, 0.6859436644795092, 0.7155569613636353, 1.3439258687459783, 1.3246194446931803, 1.3004895113079367, 1.2694288333012982, 2.009558665326211, 1.96970386169125, 1.8244250615184896, 1.6529419249494095, 1.24414521844288, 1.0943599642755901, 0.9362285212165059, 0.8607523377030725, 0.5518971596612197, 0.4872731259576552, 0.4398348869409435, 0.42902586680518967 ], "peak_layer": 21, "peak_normalized": 2.009558665326211, "population_cv": 0.4073851731752779, "rest_mean": 2.629478112120759e-06, "spike_contrast": 2.016555615555062, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.30248885297624e-06, "top_five_layers": [ 21, 22, 23, 24, 17 ], "top_five_spike_overlap": 4 }, "values": [ 3.394117584321066e-06, 2.923891997852479e-06, 2.7497233077156125e-06, 2.7733599381463137e-06, 2.5806873509282013e-06, 2.4804271561151836e-06, 2.3860582132329e-06, 2.3736845378152793e-06, 2.6584370971249882e-06, 2.584977210062789e-06, 2.541225967434002e-06, 2.539227807574207e-06, 2.073494897558703e-06, 2.07938251151063e-06, 2.0901636617054464e-06, 2.180399405915523e-06, 4.095124950254103e-06, 4.036295649711974e-06, 3.962768460041843e-06, 3.8681223486491945e-06, 6.123398634372279e-06, 6.0019556258339435e-06, 5.559271357924445e-06, 5.036738912167493e-06, 3.791079734583036e-06, 3.334663688292494e-06, 2.852815669029951e-06, 2.6228294700558763e-06, 1.6817057257867418e-06, 1.4847875036139158e-06, 1.34023673581396e-06, 1.3073001809971174e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.3425848363986006e-05, "first_to_last_ratio": 2.239424829290692, "imbalance_abs_log_ratio": 0.8062190602571265, "last_quartile_mean": 2.3856950974732172e-05, "mean": 4.4570399836629804e-05, "normalized": [ 1.1494336205459152, 1.171873145104734, 1.142800837422437, 1.0864554423282164, 1.3229603063582054, 1.3578480224061202, 1.229609154713601, 1.1284956364350298, 1.2271150761135223, 1.0975498313673546, 1.0166755209562053, 0.9402071494360641, 0.8277227843287724, 0.7884832455736381, 0.7746476228748241, 0.7477519611549666, 1.307967756375797, 1.2626631238761457, 1.1708682002256867, 1.0706732694770698, 1.7921913698445469, 1.5923449556348939, 1.3491832296364525, 1.1623630624416765, 0.9084319709996431, 0.7836885353653049, 0.6646335840154506, 0.6021591842085282, 0.3744874199934144, 0.3346428492438081, 0.3104099404736517, 0.3036621910683233 ], "peak_layer": 21, "peak_normalized": 1.7921913698445469, "population_cv": 0.35707915921975414, "rest_mean": 4.15915873654066e-05, "spike_contrast": 1.4583715367325734, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.065598718123511e-05, "top_five_layers": [ 21, 22, 6, 23, 5 ], "top_five_spike_overlap": 3 }, "values": [ 5.1230716053396463e-05, 5.223085463512689e-05, 5.0935090257553384e-05, 4.8423753469251096e-05, 5.8964869822375476e-05, 6.0519829276017845e-05, 5.48041716683656e-05, 5.02975017298013e-05, 5.46930095879361e-05, 4.8918234824668616e-05, 4.531363447313197e-05, 4.190540857962333e-05, 3.689193545142189e-05, 3.5143013519700617e-05, 3.452635428402573e-05, 3.332760388730094e-05, 5.829664587508887e-05, 5.6277400290127844e-05, 5.2186063840053976e-05, 4.7720335714984685e-05, 7.987868593772873e-05, 7.097145135048777e-05, 6.013363599777222e-05, 5.180698644835502e-05, 4.048917617183179e-05, 3.4929311368614435e-05, 2.962298458442092e-05, 2.6838475605472922e-05, 1.6691054042894393e-05, 1.4915165593265556e-05, 1.3835095160175115e-05, 1.3534345271182247e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.0632114178151824e-05, "first_to_last_ratio": 2.3114815443448227, "imbalance_abs_log_ratio": 0.837888680137748, "last_quartile_mean": 1.7578385723027168e-05, "mean": 3.283649590457571e-05, "normalized": [ 1.2128463251702764, 1.2409607981861552, 1.2097488384048114, 1.1492961072150816, 1.3482524084927348, 1.376595331959661, 1.2403426967960511, 1.121214539041085, 1.2543031022204718, 1.1257078780047793, 1.041573188661204, 0.9496272000687872, 0.8637967480657868, 0.8233044041758093, 0.8143347880182855, 0.7852282395591935, 1.2279923020188777, 1.1955877909984085, 1.0982969260190423, 1.0020729756097966, 1.7841330993237274, 1.5266975818342838, 1.2476748262094024, 1.077765966545238, 0.8974242777715684, 0.7709389519020208, 0.6829234854491848, 0.6280779276603982, 0.35252181345805783, 0.3264428428242703, 0.31417631661714684, 0.31014032171840134 ], "peak_layer": 21, "peak_normalized": 1.7841330993237274, "population_cv": 0.34914670400639497, "rest_mean": 3.097126648720371e-05, "spike_contrast": 1.385436878279194, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.290873475838452e-05, "top_five_layers": [ 21, 22, 6, 5, 9 ], "top_five_spike_overlap": 2 }, "values": [ 3.9825623389333487e-05, 4.0748804167378694e-05, 3.9723912777844816e-05, 3.773885691771284e-05, 4.427188468980603e-05, 4.520256698015146e-05, 4.072850788361393e-05, 3.6816756619373336e-05, 4.1186918679159135e-05, 3.6964302125852555e-05, 3.4201613743789494e-05, 3.118242966593243e-05, 2.8364058380248025e-05, 2.703443169593811e-05, 2.6739900931715965e-05, 2.5784143872442655e-05, 4.032296419609338e-05, 3.925891360267997e-05, 3.606422251323238e-05, 3.2904565159697086e-05, 5.858467920916155e-05, 5.0131398893427104e-05, 4.096926932106726e-05, 3.5390057746553794e-05, 2.9468268621712923e-05, 2.5314933736808598e-05, 2.242481423309073e-05, 2.0623878299375065e-05, 1.157558108388912e-05, 1.0719239071477205e-05, 1.0316449333913624e-05, 1.018392140395008e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013562132971856045, "first_to_last_ratio": 1.8937793918607297, "imbalance_abs_log_ratio": 0.638574510609612, "last_quartile_mean": 7.161411213019164e-05, "mean": 0.00012335703559074318, "normalized": [ 1.0190898279943903, 1.0323955479842792, 1.0116562111461564, 0.9775328839891945, 1.2172178061017016, 1.2992066539806704, 1.164116312982372, 1.0741536827144558, 1.1553960428140764, 1.0323218193398271, 0.9612127831166123, 0.9118644908803012, 0.7735676616433252, 0.7259226190596877, 0.7097387097391373, 0.6893988641881136, 1.4121216600785822, 1.346602611711891, 1.2404286450676882, 1.1276439912131677, 1.8527673327840795, 1.7347177459206955, 1.5317035051224266, 1.3548754374827359, 0.9638063798796889, 0.8533115017901868, 0.7100586330731438, 0.6469535195087284, 0.43178667471980076, 0.37906287870027017, 0.3347439711995271, 0.324623594073087 ], "peak_layer": 21, "peak_normalized": 1.8527673327840795, "population_cv": 0.37043820400707683, "rest_mean": 0.00011221894426314436, "spike_contrast": 1.6352205945680411, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001835027287597768, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012571190018206835, 0.00012735325435642153, 0.0001247949112439528, 0.00012058555876137689, 0.00015015238022897393, 0.0001602662814548239, 0.00014360193745233119, 0.00013250441406853497, 0.00014252623077481985, 0.0001273441594094038, 0.00011857235949719325, 0.00011248490045545623, 9.542501356918365e-05, 8.95476623554714e-05, 8.755126327741891e-05, 8.504220022587106e-05, 0.000174195141880773, 0.00016611290629953146, 0.00015301560051739216, 0.0001391028199577704, 0.000228551885811612, 0.00021398963872343302, 0.00018894640379585326, 0.0001671334175625816, 0.00011889229790540412, 0.00010526197729632258, 8.759072807151824e-05, 7.980626833159477e-05, 5.326392420101911e-05, 4.676007301895879e-05, 4.129302396904677e-05, 4.004460424766876e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b054da1b5190b8b19869952768070eae883b70fbd71c9a2eaf9205894fe44b7f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.10241243988275528, 0.1819411665201187, 0.24157075583934784, 0.26444777846336365, 0.10636615753173828, 0.13034582138061523, 0.14029134809970856, 0.15042904019355774, 0.04812335968017578, 0.07954440265893936, 0.10673297941684723, 0.14089107513427734, 0.06885477155447006, 0.10946234315633774, 0.13876035809516907, 0.1687854677438736, 0.05059739947319031, 0.07094386219978333, 0.09166282415390015, 0.12053482979536057, 0.048727843910455704, 0.07109228521585464, 0.09690473228693008, 0.11853098124265671, 0.05259029567241669, 0.08889466524124146, 0.12455403059720993, 0.1728687882423401, 0.06489895284175873, 0.14679722487926483, 0.23576539754867554, 0.35208994150161743 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6889579683265765e-05, "first_to_last_ratio": 2.125980149627833, "imbalance_abs_log_ratio": 0.754232942918421, "last_quartile_mean": 7.944373180635012e-06, "mean": 1.4253392819796318e-05, "normalized": [ 1.1153433675846105, 1.1464834343722714, 1.124956448240789, 1.0622490496205712, 1.3172283984218986, 1.3595312111704494, 1.229301131769315, 1.124519228992338, 1.1558693549986068, 1.061428019254281, 0.9912129816477457, 0.9372316548143664, 0.8026823443963541, 0.770025154892527, 0.7622460724323464, 0.7357708899991084, 1.3570894960856237, 1.294169996358889, 1.1908969211284495, 1.0892496991453142, 1.7975621100651502, 1.5821389795391794, 1.3498915859714575, 1.1839851305574904, 0.9181060564815129, 0.760619453477076, 0.6816165769744638, 0.6358942400688073, 0.4049116684425325, 0.36155600724105696, 0.3485738753445159, 0.3476594605109013 ], "peak_layer": 21, "peak_normalized": 1.7975621100651502, "population_cv": 0.34918660027907056, "rest_mean": 1.3286440611813709e-05, "spike_contrast": 1.4657751697310237, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9474934742902405e-05, "top_five_layers": [ 21, 22, 6, 17, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.5897427147137932e-05, 1.6341278751497157e-05, 1.603444616193883e-05, 1.514065297669731e-05, 1.8774973796098493e-05, 1.9377932403585874e-05, 1.752171192492824e-05, 1.6028214304242283e-05, 1.6475059965159744e-05, 1.5128950508369599e-05, 1.4128147995506879e-05, 1.3358730939216912e-05, 1.144094676419627e-05, 1.0975471013807692e-05, 1.086459269572515e-05, 1.048723152052844e-05, 1.934312967932783e-05, 1.8446313333697617e-05, 1.6974321624729782e-05, 1.5525503840763122e-05, 2.562135887274053e-05, 2.2550848370883614e-05, 1.9240535038989037e-05, 1.687580515863374e-05, 1.3086126273265108e-05, 1.0841407856787555e-05, 9.715348824101966e-06, 9.063650395546574e-06, 5.771365067630541e-06, 5.153399797563907e-06, 4.968360372004099e-06, 4.955326858180342e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.725337607349502e-05, "first_to_last_ratio": 2.130880302861834, "imbalance_abs_log_ratio": 0.7565351820700446, "last_quartile_mean": 3.1561311061523156e-05, "mean": 5.6670258345548064e-05, "normalized": [ 1.1106881622493918, 1.145498588275503, 1.1258880010928651, 1.0664271230850946, 1.3263713679598177, 1.367809856868302, 1.2291495077292092, 1.122159908308202, 1.154668923611267, 1.062750001572791, 0.9896186154790125, 0.9362122130506743, 0.8068839078557516, 0.7716166020495587, 0.7612932503122151, 0.7347861986228512, 1.357749257546284, 1.2928515294142118, 1.190927729797305, 1.089408233799192, 1.8057756999530474, 1.5765198843812338, 1.3434883429213678, 1.176024987992221, 0.9111282484717285, 0.7565791137700626, 0.678779490957478, 0.6338427293588494, 0.4073256246452393, 0.3649030178459797, 0.3518060328673549, 0.35106784815593767 ], "peak_layer": 21, "peak_normalized": 1.8057756999530474, "population_cv": 0.34914284904214865, "rest_mean": 5.286508732953937e-05, "spike_contrast": 1.4606649819879878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.721818183199502e-05, "top_five_layers": [ 21, 22, 6, 17, 23 ], "top_five_spike_overlap": 3 }, "values": [ 6.294298509601504e-05, 6.491570093203336e-05, 6.380436389008537e-05, 6.04347005719319e-05, 7.516580808442086e-05, 7.751413795631379e-05, 6.965622014831752e-05, 6.359309190884233e-05, 6.543538620462641e-05, 6.0226317145861685e-05, 5.608194260275923e-05, 5.3055387979839e-05, 4.572631951305084e-05, 4.3727712181862444e-05, 4.314268517191522e-05, 4.164052370470017e-05, 7.694400119362399e-05, 7.326623017434031e-05, 6.749018211849034e-05, 6.173704605316743e-05, 0.00010233377543045208, 8.934178913477808e-05, 7.613583147758618e-05, 6.664563989033923e-05, 5.163387322681956e-05, 4.2875533836195245e-05, 3.8466609112219885e-05, 3.59200312232133e-05, 2.308324837940745e-05, 2.0679148292401806e-05, 1.9936938770115376e-05, 1.9895105651812628e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05964096589013934, "first_to_last_ratio": 2.1308802328272587, "imbalance_abs_log_ratio": 0.756535149203545, "last_quartile_mean": 0.027988886926323175, "mean": 0.05025575193576515, "normalized": [ 1.1106881537388107, 1.1454985452622695, 1.1258879706126397, 1.0664271338250162, 1.3263713402742836, 1.3678099140413058, 1.2291494267200453, 1.122159919240132, 1.154668900806693, 1.0627500074578355, 0.9896186606157457, 0.9362122658369968, 0.8068838784002234, 0.7716165702350821, 0.7612932488495279, 0.7347862276269627, 1.3577492974867253, 1.2928515677722938, 1.1909276528528374, 1.0894083211634755, 1.805775782344081, 1.5765197105460131, 1.3434883687023758, 1.1760249355734054, 0.9111282538244218, 0.7565791648411243, 0.6787795442574317, 0.6338427117183585, 0.40732561946444507, 0.3649030144809506, 0.35180602228123725, 0.3510678691472537 ], "peak_layer": 21, "peak_normalized": 1.805775782344081, "population_cv": 0.3491428438845931, "rest_mean": 0.04688128842799752, "spike_contrast": 1.4606649512818284, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.06847785487771034, "top_five_layers": [ 21, 22, 6, 17, 23 ], "top_five_spike_overlap": 3 }, "values": [ 0.05581846833229065, 0.057567890733480453, 0.05658234655857086, 0.05359409749507904, 0.06665778905153275, 0.06874031573534012, 0.061771828681230545, 0.05639499053359032, 0.058028753846883774, 0.05340930074453354, 0.04973402991890907, 0.047050051391124725, 0.040550556033849716, 0.03877817094326019, 0.038259364664554596, 0.0369272343814373, 0.06823471188545227, 0.06497322767972946, 0.059850964695215225, 0.054749034345149994, 0.09075061976909637, 0.07922918349504471, 0.06751801818609238, 0.05910201743245125, 0.045789435505867004, 0.03802245482802391, 0.03411257639527321, 0.03185424208641052, 0.020470455288887024, 0.01833847537636757, 0.017680276185274124, 0.01764317974448204 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.6838953033347934e-06, "first_to_last_ratio": 1.2889440986362102, "imbalance_abs_log_ratio": 0.25382335500748454, "last_quartile_mean": 2.082243369727621e-06, "mean": 2.846027708613974e-06, "normalized": [ 1.1585324560759196, 1.0223734609389419, 0.9675124661430837, 0.9815131475159752, 0.9061526900553396, 0.8633991057607137, 0.8261330350267956, 0.8186399230205594, 0.8916923923720145, 0.8654164482697974, 0.843209462677348, 0.8552471278215477, 0.7079228836901937, 0.7114272483418291, 0.7214437361806822, 0.7529832577201679, 1.3512097970186538, 1.2964355644882453, 1.2560996594542384, 1.2250466776906157, 2.006533496683837, 1.883321173188558, 1.656149120836194, 1.5785540956364958, 1.1809573060310585, 0.9928361025066657, 0.906104994776819, 0.8769702899521535, 0.5241597663354778, 0.46173906989410896, 0.45174563077862806, 0.45853841311734106 ], "peak_layer": 21, "peak_normalized": 2.006533496683837, "population_cv": 0.3771990652330804, "rest_mean": 2.4975985297714942e-06, "spike_contrast": 1.8928363457980932, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.727545274363365e-06, "top_five_layers": [ 21, 22, 23, 24, 17 ], "top_five_spike_overlap": 4 }, "values": [ 3.297215471320669e-06, 2.909703198383795e-06, 2.7535672870726557e-06, 2.7934136141993804e-06, 2.5789356641325867e-06, 2.4572577785875183e-06, 2.351197508687619e-06, 2.329871904294123e-06, 2.537781256251037e-06, 2.4629991912661353e-06, 2.3997974949452328e-06, 2.4340570234926417e-06, 2.0147681425441988e-06, 2.0247416614438407e-06, 2.0532488633762114e-06, 2.143011215594015e-06, 3.8455805224657524e-06, 3.6896915389661444e-06, 3.574894435587339e-06, 3.4865167890529847e-06, 5.710649929824285e-06, 5.359984243114013e-06, 4.713446287496481e-06, 4.49260869572754e-06, 3.3610372156545054e-06, 2.8256390578462742e-06, 2.578799922048347e-06, 2.49588174483506e-06, 1.491773218731396e-06, 1.3141221870682784e-06, 1.2856805824412731e-06, 1.305013029195834e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.2885012337355874e-05, "first_to_last_ratio": 2.4318717272327546, "imbalance_abs_log_ratio": 0.888661218976389, "last_quartile_mean": 2.1746629045082955e-05, "mean": 4.23981676931362e-05, "normalized": [ 1.190270602106981, 1.2242785076575162, 1.2015505398967856, 1.1384753861662993, 1.3837446167726861, 1.412880509028644, 1.2711237134722582, 1.1564115683962857, 1.205924025589364, 1.101554226030039, 1.0185042468172618, 0.9526077357578042, 0.8449377006342662, 0.8080550669856646, 0.796734028555351, 0.7697863378322973, 1.310756743636259, 1.254353365068151, 1.1507733156032836, 1.0555102681924855, 1.792017786366534, 1.51664778521082, 1.252309144484671, 1.0874779001939108, 0.8458376244707143, 0.7039938225479899, 0.635304784671484, 0.596370594103779, 0.35431838957072104, 0.3268559755339857, 0.3200335480760464, 0.32060014056966063 ], "peak_layer": 21, "peak_normalized": 1.792017786366534, "population_cv": 0.35909580795741874, "rest_mean": 4.0051679981708595e-05, "spike_contrast": 1.374953593956486, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.506920133484528e-05, "top_five_layers": [ 21, 22, 6, 5, 17 ], "top_five_spike_overlap": 2 }, "values": [ 5.046529258834198e-05, 5.1907165470765904e-05, 5.094354128232226e-05, 4.8269270337186754e-05, 5.866823630640283e-05, 5.990354475216009e-05, 5.389331636251882e-05, 4.902973159914836e-05, 5.112896906211972e-05, 4.6703880798304453e-05, 4.318271385272965e-05, 4.038882252643816e-05, 3.582381032174453e-05, 3.4260054235346615e-05, 3.378006294951774e-05, 3.263753023929894e-05, 5.557368422159925e-05, 5.318228431860916e-05, 4.8790680011734366e-05, 4.475170135265216e-05, 7.597827061545104e-05, 6.430308712879196e-05, 5.309561311150901e-05, 4.610707037500106e-05, 3.586196544347331e-05, 2.9848048143321648e-05, 2.6935758796753362e-05, 2.5285020456067286e-05, 1.5022450497781392e-05, 1.3858094462193549e-05, 1.3568836038757581e-05, 1.359285852231551e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.03796584578231e-05, "first_to_last_ratio": 2.4739007999741456, "imbalance_abs_log_ratio": 0.9057961761470462, "last_quartile_mean": 1.6322262581525138e-05, "mean": 3.154074499889248e-05, "normalized": [ 1.2508863516144793, 1.2841867971630014, 1.2593428954345096, 1.1937167612399988, 1.4009066408475082, 1.420515275619532, 1.2780975365837242, 1.1542512752891054, 1.235598435522957, 1.124789879408544, 1.0344513335895396, 0.9505323884005586, 0.8790314147700448, 0.8375394240979268, 0.8329084925799662, 0.802864330251641, 1.2379667568434787, 1.1905528094480917, 1.0824881288459707, 0.9955427197632982, 1.7778790802657947, 1.461709510411412, 1.1598120379240673, 1.0144482285055212, 0.8299461589475229, 0.7167094189126102, 0.6574294700400573, 0.6234038104311064, 0.3406256334471781, 0.3271495125781152, 0.3224606792775168, 0.3222568119452214 ], "peak_layer": 21, "peak_normalized": 1.7778790802657947, "population_cv": 0.35301095948018335, "rest_mean": 3.0087773834734604e-05, "spike_contrast": 1.30906292707755, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.9386789285345e-05, "top_five_layers": [ 21, 22, 6, 5, 2 ], "top_five_spike_overlap": 2 }, "values": [ 3.945388743886724e-05, 4.050420830026269e-05, 3.9720613131066784e-05, 3.765071596717462e-05, 4.4185639126226306e-05, 4.4804110075347126e-05, 4.03121484850999e-05, 3.640594513854012e-05, 3.897169517586008e-05, 3.547671076375991e-05, 3.2627365726511925e-05, 2.9980499675730243e-05, 2.772530569927767e-05, 2.641661740199197e-05, 2.6270554371876642e-05, 2.5322939109173603e-05, 3.904639379470609e-05, 3.755092257051729e-05, 3.4142482036259025e-05, 3.1400159059558064e-05, 5.607563070952892e-05, 4.610340693034232e-05, 3.658133573480882e-05, 3.199645288987085e-05, 2.6177120162174106e-05, 2.2605549020227045e-05, 2.073581526929047e-05, 1.9662620616145432e-05, 1.0743586244643666e-05, 1.0318539352738298e-05, 1.017065005726181e-05, 1.0164219929720275e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013370490523811895, "first_to_last_ratio": 2.067929402982526, "imbalance_abs_log_ratio": 0.7265478182407109, "last_quartile_mean": 6.465641672548372e-05, "mean": 0.00011615701851042104, "normalized": [ 1.0560644548880596, 1.0932324234962196, 1.0704146702692696, 1.0356495330841202, 1.2890613053020747, 1.349240331041549, 1.2161603019292984, 1.0987401440739695, 1.1302068376440797, 1.052621189942207, 0.9794612370099226, 0.9384889523526194, 0.795246875601738, 0.7437305707818854, 0.7363409872103527, 0.7207683094190416, 1.4084141263671424, 1.3357660592114178, 1.2156581878342292, 1.0939775768488822, 1.8651920040475745, 1.6523968523362491, 1.4344972498256239, 1.2356342609168542, 0.9104609510825358, 0.7537285042826567, 0.6882282342205499, 0.6516164997895624, 0.4042698411477325, 0.355151701146589, 0.3437385024432238, 0.34584132445276966 ], "peak_layer": 21, "peak_normalized": 1.8651920040475745, "population_cv": 0.36353364558210677, "rest_mean": 0.00010713040790977646, "spike_contrast": 1.539252197123888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016490071575390176, "top_five_layers": [ 21, 22, 23, 17, 6 ], "top_five_spike_overlap": 3 }, "values": [ 0.00012266929843463004, 0.00012698661885224283, 0.00012433617666829377, 0.00012029796198476106, 0.00014973351790104061, 0.00015672373410779983, 0.00014126555470284075, 0.00012762637925334275, 0.0001312814565608278, 0.00012226933904457837, 0.00011377129703760147, 0.00010901207861024886, 9.237350604962558e-05, 8.638952567707747e-05, 8.553117368137464e-05, 8.37222978589125e-05, 0.00016359718574676663, 0.00015515860286541283, 0.00014120723062660545, 0.00012707317364402115, 0.00021665514213964343, 0.00019193749176338315, 0.00016662692360114306, 0.00014352759171742946, 0.00010575642954790965, 8.755085582379252e-05, 7.99425397417508e-05, 7.568982982775196e-05, 4.695877942140214e-05, 4.125336272409186e-05, 3.992763959104195e-05, 4.01718971261289e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4d1314009c174d2647fbc47c0cd287abca38783c7de30eced47b74a084c87fbc", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05571148172020912, 0.061161335557699203, 0.09595561027526855, 0.12880638241767883, 0.13726800680160522, 0.10176635533571243, 0.09925490617752075, 0.10055620223283768, 0.0990053340792656, 0.07309409230947495, 0.07328634709119797, 0.07434380799531937, 0.07861478626728058, 0.06236656382679939, 0.06600815057754517, 0.07462523877620697, 0.07498960196971893, 0.05233807489275932, 0.05626988410949707, 0.061378348618745804, 0.06856362521648407, 0.04801172763109207, 0.0518198125064373, 0.05105668306350708, 0.054886870086193085, 0.04603559896349907, 0.04718422889709473, 0.05210597813129425, 0.05379685014486313, 0.05137190222740173, 0.05757243186235428, 0.06228902190923691 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.289336674100923e-06, "first_to_last_ratio": 2.2865411488778773, "imbalance_abs_log_ratio": 0.8270402604018311, "last_quartile_mean": 4.0626151331935034e-06, "mean": 9.477520439560294e-06, "normalized": [ 1.252991954527522, 0.7394811736091712, 0.5152207040408479, 0.7320401747081184, 0.45740983320689066, 1.2545020339537147, 1.465347353652639, 1.4241603595420207, 1.5020047812281965, 1.3115942831695324, 0.9033867836813385, 0.9397941307418568, 0.6815998176876099, 0.7241591358217578, 0.9921275569566015, 0.8513079482595329, 0.987677640009984, 1.6262023844874374, 1.2990996618268078, 1.8103875474840807, 1.9614463506839017, 2.117788128754767, 1.9620662739867234, 1.0589398388998983, 0.9090157064636322, 0.8549381461887039, 0.37264179907278366, 0.3857075463499887, 0.23244225762357623, 0.33699601723381234, 0.16357771526793677, 0.17394496087861533 ], "peak_layer": 22, "peak_normalized": 2.117788128754767, "population_cv": 0.5364323230626372, "rest_mean": 8.421213473647421e-06, "spike_contrast": 1.8027779610382346, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.5181578055489808e-05, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.1875256859639194e-05, 7.0084479375509545e-06, 4.88301475343178e-06, 6.937925718375482e-06, 4.335111043474171e-06, 1.1889568668266293e-05, 1.3887859495298471e-05, 1.349750891677104e-05, 1.423528101440752e-05, 1.2430661627149675e-05, 8.56186670716852e-06, 8.906918083084747e-06, 6.459876203734893e-06, 6.863233011245029e-06, 9.40290919970721e-06, 8.06828847998986e-06, 9.360735020891298e-06, 1.5412366337841377e-05, 1.2312243597989436e-05, 1.7157984984805807e-05, 1.8589647879707627e-05, 2.007138027693145e-05, 1.859552321548108e-05, 1.003612396743847e-05, 8.615214937890414e-06, 8.102693755063228e-06, 3.5317202673468273e-06, 3.6555511542246677e-06, 2.2029762476449832e-06, 3.1938866413838696e-06, 1.5503111399084446e-06, 1.6485669220855925e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.385339800843212e-05, "first_to_last_ratio": 2.08046689071163, "imbalance_abs_log_ratio": 0.7325923352126326, "last_quartile_mean": 1.627201959308877e-05, "mean": 3.714502105367501e-05, "normalized": [ 0.6435910429764604, 0.6409303103278117, 0.5081801030667545, 0.7165876903215328, 0.47047927357980707, 1.3039514496969595, 1.5507865803288083, 1.4565698873921415, 1.551285486064157, 1.3082956708978086, 0.9732102467191055, 0.9041604784923716, 0.6922959073858851, 0.7633837554908558, 1.007898690427689, 0.8632027073532546, 1.0068526925373231, 1.6741376444679952, 1.3331202957363155, 1.7806470652336404, 2.0225428919697093, 2.301977702545363, 2.002199408280622, 1.019174508509882, 0.9428974629120086, 0.8578527415516407, 0.38583353479489596, 0.3927921630823954, 0.2389950189630043, 0.3440899675346293, 0.16786811733193024, 0.1742095040272426 ], "peak_layer": 22, "peak_normalized": 2.301977702545363, "population_cv": 0.562210067540178, "rest_mean": 3.2620493382435394e-05, "spike_contrast": 1.8876927996290056, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.157747047836892e-05, "top_five_layers": [ 22, 21, 23, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 2.390620284131728e-05, 2.380736987106502e-05, 1.8876360627473332e-05, 2.6617664843797684e-05, 1.747596252243966e-05, 4.843530405196361e-05, 5.760400017607026e-05, 5.4104319133330137e-05, 5.762253204011358e-05, 4.859667023993097e-05, 3.6149915104033425e-05, 3.358506000950001e-05, 2.5715346055221744e-05, 2.8355905669741333e-05, 3.7438418075907975e-05, 3.206368273822591e-05, 3.7399564462248236e-05, 6.218587805051357e-05, 4.9518781452206895e-05, 6.614217272726819e-05, 7.512739830417559e-05, 8.550701022613794e-05, 7.437173917423934e-05, 3.7857258575968444e-05, 3.502394611132331e-05, 3.186495814588852e-05, 1.4331794773170259e-05, 1.4590273167414125e-05, 8.877475011104252e-06, 1.2781229088432156e-05, 6.235464752535336e-06, 6.471015694842208e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03002153232228011, "first_to_last_ratio": 2.0804669230521093, "imbalance_abs_log_ratio": 0.7325923507574504, "last_quartile_mean": 0.014430189679842442, "mean": 0.03294057691527996, "normalized": [ 0.6435910397676826, 0.6409302858979954, 0.5081800703155225, 0.716587695511193, 0.4704792716879319, 1.3039513599328192, 1.5507866493520404, 1.456569894837028, 1.5512854947592953, 1.3082956463699744, 0.973210295631563, 0.9041604672039659, 0.6922959258926152, 0.7633837713422348, 1.0078987680340359, 0.8632027260248254, 1.0068526741738888, 1.6741375136914436, 1.3331203018554725, 1.78064694542985, 2.0225429754607553, 2.301977829207771, 2.0021994467984428, 1.0191745289341387, 0.9428974359364356, 0.8578527192959636, 0.3858335126274743, 0.39279215725797884, 0.2389950138470689, 0.3440899570497142, 0.16786812365093953, 0.17420950222193982 ], "peak_layer": 22, "peak_normalized": 2.301977829207771, "population_cv": 0.5622100766374781, "rest_mean": 0.028928180138959928, "spike_contrast": 1.887692874045117, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.054607519507408143, "top_five_layers": [ 22, 21, 23, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.021200260147452354, 0.021112613379955292, 0.016739744693040848, 0.02360481210052967, 0.015497858636081219, 0.04295291006565094, 0.05108380690217018, 0.04798025265336037, 0.0511002391576767, 0.043096013367176056, 0.03205810859799385, 0.029783567413687706, 0.02280462719500065, 0.025146301835775375, 0.033200766891241074, 0.028434395790100098, 0.0331663079559803, 0.05514705553650856, 0.04391375184059143, 0.05865553766489029, 0.0666237324476242, 0.07582847774028778, 0.06595360487699509, 0.03357219696044922, 0.031059585511684418, 0.02825816348195076, 0.01270957849919796, 0.012938800267875195, 0.007872633635997772, 0.011334521695971489, 0.005529672838747501, 0.005738561507314444 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.777704892764632e-06, "first_to_last_ratio": 1.0317973970310554, "imbalance_abs_log_ratio": 0.03130232707959694, "last_quartile_mean": 2.6921030240600885e-06, "mean": 3.2474394480175306e-06, "normalized": [ 3.264922808244637, 1.0119989909820577, 0.406734804751791, 0.3594970166401698, 0.3085255046215715, 0.44136542394078937, 0.4976088478313049, 0.5521652291156375, 0.5596032700860724, 0.7626434513115219, 0.6804286383958748, 0.8574740801621124, 0.5510556809031377, 0.657880030915651, 0.5700052058099263, 0.6746699382392725, 0.8730218984285152, 1.4178722452350283, 1.4067082871744077, 1.5468786674281858, 1.5912638166777024, 2.4472781271242594, 2.129464926083038, 1.7989929191022997, 1.363241191573045, 1.2762479066181045, 1.0041585663694674, 1.0201870464948222, 0.6062702503600963, 0.5376439897532335, 0.43319266698828435, 0.39099857263798254 ], "peak_layer": 1, "peak_normalized": 3.264922808244637, "population_cv": 0.670834276414478, "rest_mean": 2.726617396917761e-06, "spike_contrast": 2.2224894958883965, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.059878523956286e-06, "top_five_layers": [ 1, 22, 23, 24, 21 ], "top_five_spike_overlap": 4 }, "values": [ 1.060263912222581e-05, 3.2864054446690716e-06, 1.3208466498326743e-06, 1.167444793281902e-06, 1.0019178944276064e-06, 1.4333074886963004e-06, 1.615954602129932e-06, 1.793123146853759e-06, 1.8172777345171198e-06, 2.476638428561273e-06, 2.2096508018876193e-06, 2.78459515357099e-06, 1.78951995621901e-06, 2.1364255644584773e-06, 1.8510573909225059e-06, 2.1909497718297644e-06, 2.835085751939914e-06, 4.604454261425417e-06, 4.5681999836233445e-06, 5.023394805903081e-06, 5.167532890482107e-06, 7.947387530293781e-06, 6.9153084041317925e-06, 5.842120572197018e-06, 4.42704322267673e-06, 4.144537797401426e-06, 3.2609441404929385e-06, 3.3129956591437804e-06, 1.968825927178841e-06, 1.7459663013141835e-06, 1.406766955369676e-06, 1.2697441889031325e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4231409383901337e-05, "first_to_last_ratio": 2.412573321332518, "imbalance_abs_log_ratio": 0.8806939460313521, "last_quartile_mean": 1.004380226277135e-05, "mean": 2.5713559296036692e-05, "normalized": [ 0.6136241747208916, 0.6528682314803166, 0.5678169227836256, 0.7498796676883942, 0.5334908499704412, 1.3465695630143542, 1.6172632261416102, 1.457361031568912, 1.4228430752167622, 1.3681707330388233, 0.9853905554776219, 0.9155538605941198, 0.6950712915766559, 0.7940335378691747, 1.0577609619611685, 0.881295451045459, 1.0180543332344463, 1.8064117968549576, 1.3937187971362992, 1.682480285033005, 1.9772967331635165, 2.3224003214146545, 1.9930145593470021, 1.0228032828152094, 0.9037764911003984, 0.7535194773585814, 0.3571273114497246, 0.3679098930110221, 0.19844991746454535, 0.2567840165451007, 0.14387571001082, 0.14338393991238602 ], "peak_layer": 22, "peak_normalized": 2.3224003214146545, "population_cv": 0.5708513691298637, "rest_mean": 2.2647654111130612e-05, "spike_contrast": 1.8663940683267242, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2269447294529525e-05, "top_five_layers": [ 22, 23, 21, 18, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.5778461602167226e-05, 1.678756598266773e-05, 1.4600594113289844e-05, 1.9282075299997814e-05, 1.3717948604607955e-05, 3.462509630480781e-05, 4.1585593862691894e-05, 3.747393930098042e-05, 3.658635978354141e-05, 3.518053927109577e-05, 2.5337898478028364e-05, 2.354214848310221e-05, 1.7872756870929152e-05, 2.041742845904082e-05, 2.719879921642132e-05, 2.2661242837784812e-05, 2.6177800464211032e-05, 4.644927685149014e-05, 3.583747093216516e-05, 4.326255657360889e-05, 5.0843336794059724e-05, 5.971717837383039e-05, 5.124749804963358e-05, 2.6299912860849872e-05, 2.323931039427407e-05, 1.937566776177846e-05, 9.183014299196657e-06, 9.460272849537432e-06, 5.1028537200181745e-06, 6.602831035706913e-06, 3.6995566006226e-06, 3.6869114410365e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7024314388436323e-05, "first_to_last_ratio": 2.80437609780141, "imbalance_abs_log_ratio": 1.0311810892083095, "last_quartile_mean": 6.070624550602588e-06, "mean": 1.751632886026755e-05, "normalized": [ 0.584240566149076, 0.6817235247296635, 0.6361546768283869, 0.7650115500840945, 0.6079996047256336, 1.3836573287680065, 1.6554473132638503, 1.4610542070855017, 1.2966582857771938, 1.3930304105863356, 0.9987773633166749, 0.9034199114794684, 0.7206853676335052, 0.821798500240951, 1.1153051447773232, 0.8912570241764348, 1.0577423140442908, 1.92155965677437, 1.4320706567338384, 1.5861347958986807, 1.9171730212528781, 2.3573454464224612, 1.9827077484178592, 1.0564899390528322, 0.8322707332281934, 0.6678826522442516, 0.33914335537769175, 0.32515959168264075, 0.17199332542575016, 0.18497778526153036, 0.13366747042309535, 0.11746072813753591 ], "peak_layer": 22, "peak_normalized": 2.3573454464224612, "population_cv": 0.5792420506711731, "rest_mean": 1.547536067779152e-05, "spike_contrast": 1.8440640990417727, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8537557045638096e-05, "top_five_layers": [ 22, 23, 18, 21, 7 ], "top_five_spike_overlap": 3 }, "values": [ 1.0233749890176114e-05, 1.1941293450945523e-05, 1.114309452532325e-05, 1.3400193893176038e-05, 1.0649921023286879e-05, 2.4236596800619736e-05, 2.8997359549975954e-05, 2.5592305973987095e-05, 2.2712692953064106e-05, 2.4400778784183785e-05, 1.74949127540458e-05, 1.5824600268388167e-05, 1.2623761904251296e-05, 1.439489278709516e-05, 1.9536051695467904e-05, 1.561155113449786e-05, 1.852776222222019e-05, 3.3658670872682706e-05, 2.5084620574489236e-05, 2.778325870167464e-05, 3.358183312229812e-05, 4.129203807679005e-05, 3.472976095508784e-05, 1.8505825210013427e-05, 1.457832786400104e-05, 1.1698852176778018e-05, 5.940546543570235e-06, 5.695602339983452e-06, 3.012691649928456e-06, 3.2401317184849177e-06, 2.341363369851024e-06, 2.057480742223561e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.760827909602085e-05, "first_to_last_ratio": 2.1482842886109816, "imbalance_abs_log_ratio": 0.7646695182666318, "last_quartile_mean": 3.1470825092583254e-05, "mean": 7.373131964527602e-05, "normalized": [ 0.6107383587992412, 0.6200732857988205, 0.49856563619679256, 0.7443981105032205, 0.46530796044932227, 1.3039254699053608, 1.601092740237933, 1.4915359307012412, 1.5070070061189442, 1.3672490492442442, 0.9793783719831628, 0.9514935883546256, 0.6639415020375616, 0.7664323809011713, 0.9882247241637759, 0.8679526231420076, 0.9882466315784014, 1.659918490517302, 1.324670508689015, 1.8045145321525453, 2.0232513831671906, 2.378151302734961, 2.00069385121045, 0.978587238005451, 0.9661377082112428, 0.817626258962968, 0.36492362259091654, 0.41187609689456905, 0.22193259512312896, 0.3124952321103628, 0.15168754329397788, 0.16797026622009212 ], "peak_layer": 22, "peak_normalized": 2.378151302734961, "population_cv": 0.5750991325799932, "rest_mean": 6.459185429034901e-05, "spike_contrast": 1.9055720557053664, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012308443256188185, "top_five_layers": [ 22, 21, 23, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.503054515225813e-05, 4.571882163872942e-05, 3.675990228657611e-05, 5.488545502885245e-05, 3.430776996538043e-05, 9.61401456152089e-05, 0.00011805068061221391, 0.00010997291246894747, 0.0001111136152758263, 0.0001008090766845271, 7.221085979836062e-05, 7.015487790340558e-05, 4.895328311249614e-05, 5.65100708627142e-05, 7.286311301868409e-05, 6.399529229383916e-05, 7.286472828127444e-05, 0.00012238798080943525, 9.766970470082015e-05, 0.00013304923777468503, 0.00014917699445504695, 0.000175344233866781, 0.000147513797855936, 7.215252844616771e-05, 7.123460818547755e-05, 6.028466304996982e-05, 2.6906300263362937e-05, 3.0368168154382147e-05, 1.6363383110729046e-05, 2.304068584635388e-05, 1.1184122740814928e-05, 1.238466938957572e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "fbd2d69d0c904dc853dab2fd491c5cf23fa8369610dbace5a1307601df767adc", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04887383058667183, 0.06501953303813934, 0.11691275984048843, 0.07614059001207352, 0.09499973803758621, 0.09968593716621399, 0.09547581523656845, 0.09975490719079971, 0.06202486902475357, 0.06455004960298538, 0.062386661767959595, 0.07036563754081726, 0.05747222527861595, 0.057312749326229095, 0.06149744614958763, 0.06584152579307556, 0.04952141270041466, 0.04880477488040924, 0.0513896569609642, 0.057161398231983185, 0.04507989436388016, 0.04335150495171547, 0.047289349138736725, 0.05114771053195, 0.042588189244270325, 0.044108107686042786, 0.05026337504386902, 0.05349528789520264, 0.04645879194140434, 0.05323651432991028, 0.05820684880018234, 0.058245208114385605 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.5842610063573375e-06, "first_to_last_ratio": 0.31018923275847604, "imbalance_abs_log_ratio": 1.1705727394853376, "last_quartile_mean": 1.4778917261537572e-05, "mean": 1.149474866934952e-05, "normalized": [ 0.964044714824398, 0.4099629685278065, 0.15847344197468682, 0.44879991396551794, 0.2564005888362708, 0.21248149821699294, 0.26315959961471164, 0.47718531434582095, 0.8325875353069343, 0.6265632148302367, 0.6147981550559392, 0.5305901990282497, 0.5691137397437396, 0.5683891347333625, 0.5227011984642989, 0.5253372083161463, 1.2593919921808647, 1.2898533329037387, 1.2925988880315054, 1.3253277249748845, 2.110587665830833, 2.2010040974041387, 2.209732114295895, 2.045233009603335, 1.9419589930337084, 1.7718838003851192, 1.545352082047033, 1.491806952509735, 1.0189371444399558, 0.9016019294848561, 0.8000663161346, 0.8140755309546854 ], "peak_layer": 23, "peak_normalized": 2.209732114295895, "population_cv": 0.6223339325251772, "rest_mean": 9.14960031439949e-06, "spike_contrast": 2.6403940014799714, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4158549786079674e-05, "top_five_layers": [ 23, 22, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.1081451702921186e-05, 4.712421286967583e-06, 1.8216123862657696e-06, 5.158842213859316e-06, 2.9472603273461573e-06, 2.442421418891172e-06, 3.024953457497759e-06, 5.4851252571097575e-06, 9.570384463586379e-06, 7.20218667993322e-06, 7.066950274747796e-06, 6.099000984249869e-06, 6.54181940262788e-06, 6.533490250149043e-06, 6.008318905514898e-06, 6.038619176251814e-06, 1.4476394426310435e-05, 1.4826539882051293e-05, 1.4858099348202813e-05, 1.523430910310708e-05, 2.4260674763354473e-05, 2.5299988919869065e-05, 2.5400315280421637e-05, 2.3509439415647648e-05, 2.232233055110555e-05, 2.0367358956718817e-05, 1.7763433788786642e-05, 1.7147945982287638e-05, 1.171242638520198e-05, 1.0363687579229008e-05, 9.196561222779565e-06, 9.357593626191374e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.844370848402832e-05, "first_to_last_ratio": 0.3140073886393369, "imbalance_abs_log_ratio": 1.158338762652201, "last_quartile_mean": 5.8736542996484786e-05, "mean": 4.581337286424514e-05, "normalized": [ 0.983221977853255, 0.4150243087528777, 0.15746378077315915, 0.45454211061157807, 0.2523150793847112, 0.21218614843533373, 0.2635465417903775, 0.4823681378821299, 0.8343480408888806, 0.6208313208522753, 0.6128114426671225, 0.5334391065031241, 0.5690611192260892, 0.5657304418592991, 0.525481722937821, 0.5279767829162021, 1.2652030229954638, 1.2822561927565206, 1.2949405359606927, 1.3366411232201898, 2.107649239013807, 2.207816441558945, 2.189332645277925, 2.049149351497267, 1.941094009743254, 1.7640868530853437, 1.5394721367578272, 1.480393360068856, 1.0182100711985076, 0.90044042915216, 0.8028775926136404, 0.8100889317653636 ], "peak_layer": 22, "peak_normalized": 2.207816441558945, "population_cv": 0.6207014414064956, "rest_mean": 3.648943235627181e-05, "spike_contrast": 2.635356194867543, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.616265160730108e-05, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 4.504471507971175e-05, 1.9013663404621184e-05, 7.213946901174495e-06, 2.0824107195949182e-05, 1.1559404811123386e-05, 9.72096313489601e-06, 1.2073955986124929e-05, 2.2098911358625628e-05, 3.8224297895794734e-05, 2.8442376788007095e-05, 2.8074959118384868e-05, 2.44386446865974e-05, 2.6070609237649478e-05, 2.5918019673554227e-05, 2.4074090106296353e-05, 2.4188397219404578e-05, 5.79632178414613e-05, 5.874448106624186e-05, 5.9325593610992655e-05, 6.123603816376999e-05, 9.655852045398206e-05, 0.00010114751785295084, 0.00010030071280198172, 9.387844329467043e-05, 8.892806363292038e-05, 8.081876876531169e-05, 7.052841101540253e-05, 6.78218129905872e-05, 4.6647637645946816e-05, 4.125221312278882e-05, 3.678253051475622e-05, 3.711290628416464e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.016356064879801124, "first_to_last_ratio": 0.3140073792809234, "imbalance_abs_log_ratio": 1.1583387924553648, "last_quartile_mean": 0.05208815447986126, "mean": 0.040627757538459264, "normalized": [ 0.9832219159051098, 0.4150243255028214, 0.15746378595866897, 0.4545420901919184, 0.25231507934570285, 0.212186133434977, 0.26354653718783205, 0.48236814287311225, 0.8343480548464586, 0.6208312921874214, 0.6128114355572287, 0.5334391169468627, 0.5690611113916649, 0.5657304465211455, 0.5254817034060001, 0.5279767678755438, 1.2652029871996548, 1.282256277419591, 1.2949405686374758, 1.3366410906562367, 2.10764920469435, 2.2078163557568278, 2.1893326511557634, 2.049149474394762, 1.94109395479328, 1.7640869604276401, 1.5394721077288682, 1.480393334684612, 1.0182101267471966, 0.9004404430197213, 0.8028775630631926, 0.8100889604883591 ], "peak_layer": 22, "peak_normalized": 2.2078163557568278, "population_cv": 0.6207014436535331, "rest_mean": 0.03235919385527571, "spike_contrast": 2.6353561775689003, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08527800142765045, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03994610160589218, 0.016861507669091225, 0.006397400517016649, 0.018467025831341743, 0.010250995866954327, 0.00862064678221941, 0.010707304812967777, 0.019597535952925682, 0.03389769047498703, 0.025222983211278915, 0.024897154420614243, 0.021672435104846954, 0.023119676858186722, 0.02298435941338539, 0.021349143236875534, 0.02145051211118698, 0.05140236020088196, 0.0520951971411705, 0.05261053144931793, 0.05430473014712334, 0.08562906086444855, 0.08969862759113312, 0.08894767612218857, 0.08325234800577164, 0.07886229455471039, 0.07167089730501175, 0.0625452995300293, 0.0601450614631176, 0.04136759415268898, 0.03658287599682808, 0.032619114965200424, 0.03291209787130356 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.079009859239477e-06, "first_to_last_ratio": 0.27720812628615055, "imbalance_abs_log_ratio": 1.2829866964894792, "last_quartile_mean": 3.892417851147911e-06, "mean": 2.952973377823298e-06, "normalized": [ 0.9923364246284452, 0.41483623106591405, 0.15635970486874604, 0.4924804983623727, 0.23205015211521984, 0.1632731916136273, 0.19761849511604435, 0.2742273866813069, 0.647135722646919, 0.5558013269361449, 0.5842350348266575, 0.48137427600621085, 0.5660568730264243, 0.5692692385054805, 0.5090179437241957, 0.5218754519537611, 1.2935636777127382, 1.3103201829927758, 1.3424099585692733, 1.3728171691781692, 2.133225965099078, 2.191548108133119, 2.305749086881416, 2.147337119746363, 1.9823368774471204, 1.787241267304806, 1.6332376019052974, 1.5389626829092304, 1.0508228818507006, 0.9064559278972918, 0.8245776443221877, 0.821445895972963 ], "peak_layer": 23, "peak_normalized": 2.305749086881416, "population_cv": 0.6589200861025274, "rest_mean": 2.3229841609143612e-06, "spike_contrast": 2.735668738537661, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.354915149131557e-06, "top_five_layers": [ 23, 22, 24, 21, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.9303430437721545e-06, 1.2250003464941983e-06, 4.6172604584171495e-07, 1.4542818007612368e-06, 6.852379215160909e-07, 4.821413881472836e-07, 5.835621550431824e-07, 8.097861723399546e-07, 1.9109745608147932e-06, 1.641266521801299e-06, 1.725230504234787e-06, 1.421485421815305e-06, 1.671550876380934e-06, 1.6810369061204256e-06, 1.5031164366519079e-06, 1.5410843161589582e-06, 3.819859102804912e-06, 3.869340616802219e-06, 3.964100869779941e-06, 4.053892553201877e-06, 6.299359483818989e-06, 6.4715832195361145e-06, 6.8088156695012e-06, 6.341029347822769e-06, 5.853788024978712e-06, 5.277675882098265e-06, 4.822907158086309e-06, 4.544515832094476e-06, 3.1030519949126756e-06, 2.6767402232508175e-06, 2.434955831631669e-06, 2.4257078621303663e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.3062004313724174e-05, "first_to_last_ratio": 0.3462796036926242, "imbalance_abs_log_ratio": 1.0605087271060973, "last_quartile_mean": 3.7720975115007604e-05, "mean": 3.0910587128119005e-05, "normalized": [ 1.044258273718076, 0.4193917891961965, 0.1513346310446105, 0.48311870475518875, 0.26867569695647586, 0.22205928542141973, 0.2887419879047208, 0.503010044141552, 0.8687652035040012, 0.6835300441309964, 0.6794437217438256, 0.5876684752752086, 0.6255961849481598, 0.6155440908016775, 0.5767443891193211, 0.585169369008815, 1.2725148048203212, 1.2730810288123915, 1.2985548706938432, 1.36266128932131, 2.142396449976532, 2.2433771027099114, 2.089704311420349, 1.9520549176365862, 1.910920056896515, 1.6930503405879038, 1.4702439655082709, 1.4111274031667742, 0.9417698480554898, 0.8189675040741922, 0.7471564493409493, 0.7693677653084159 ], "peak_layer": 22, "peak_normalized": 2.2433771027099114, "population_cv": 0.5973036170926191, "rest_mean": 2.4798931143002982e-05, "spike_contrast": 2.577269523399549, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.391352944774553e-05, "top_five_layers": [ 22, 21, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.227863635402173e-05, 1.2963646440766752e-05, 4.677842298406176e-06, 1.4933482816559263e-05, 8.304923539981246e-06, 6.863982889626641e-06, 8.925184374675155e-06, 1.554833579575643e-05, 2.6854042516788468e-05, 2.112831498379819e-05, 2.1002004359615967e-05, 1.8165177607443184e-05, 1.9337545381858945e-05, 1.902682924992405e-05, 1.7827507690526545e-05, 1.8087928765453398e-05, 3.933417974621989e-05, 3.9351682062260807e-05, 4.013909347122535e-05, 4.212066050968133e-05, 6.622273212997243e-05, 6.934410339454189e-05, 6.459398719016463e-05, 6.033916361047886e-05, 5.906766091356985e-05, 5.233318006503396e-05, 4.54461041954346e-05, 4.361877654446289e-05, 2.9110658942954615e-05, 2.5314766389783472e-05, 2.3095044525689445e-05, 2.3781609343132004e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.172314918259872e-06, "first_to_last_ratio": 0.3673647725037251, "imbalance_abs_log_ratio": 1.0013999940215956, "last_quartile_mean": 2.4967867375380592e-05, "mean": 2.0697781202727583e-05, "normalized": [ 1.111303541294376, 0.4345514774860271, 0.14873123246072173, 0.515966262987765, 0.28568711280472064, 0.23243024785129948, 0.309127325753849, 0.5074388141476497, 0.8970449242709069, 0.738367224211955, 0.7462688121005764, 0.642179562850806, 0.6755482383061984, 0.6507702437283572, 0.6234304508304926, 0.6319968759160601, 1.225562031333336, 1.197045832524259, 1.2456604157243978, 1.337071883147354, 2.1461057208048535, 2.2767618292912672, 1.9401568217351242, 1.8303413600277973, 1.8918927171524897, 1.644370796755471, 1.4630724614723636, 1.4103465215309783, 0.9354546739436189, 0.7941395510132133, 0.7495417621189062, 0.7616332744228086 ], "peak_layer": 22, "peak_normalized": 2.2767618292912672, "population_cv": 0.5708172615963157, "rest_mean": 1.6799500953210452e-05, "spike_contrast": 2.485103257912062, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.174849455012009e-05, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.3001517547527328e-05, 8.994251402327791e-06, 3.0784065074840328e-06, 1.067935681930976e-05, 5.9130893532710616e-06, 4.810790414921939e-06, 6.3982497522374615e-06, 1.05028575489996e-05, 1.8566839571576566e-05, 1.5282563254004344e-05, 1.5446108591277152e-05, 1.329169208474923e-05, 1.3982349628349766e-05, 1.346950011793524e-05, 1.2903627066407353e-05, 1.3080933058517985e-05, 2.5366414774907753e-05, 2.4776192731224e-05, 2.578240673756227e-05, 2.7674421289702877e-05, 4.441962664714083e-05, 4.7123918193392456e-05, 4.015694139525294e-05, 3.788400499615818e-05, 3.9157981518656015e-05, 3.403482696739957e-05, 3.0282353691291064e-05, 2.9191043722676113e-05, 1.9361836166353896e-05, 1.643692667130381e-05, 1.5513851394644007e-05, 1.5764118870720267e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.607548808304273e-05, "first_to_last_ratio": 0.29287710552991697, "imbalance_abs_log_ratio": 1.228002193004033, "last_quartile_mean": 0.00012317619712121086, "mean": 9.582550958953107e-05, "normalized": [ 0.9271980053828993, 0.37240108217726614, 0.14097411813076574, 0.43019423957314057, 0.23484470025293644, 0.20099018929480036, 0.24920840443437586, 0.45595414361828196, 0.7897161290201836, 0.5997815779888516, 0.573550343218901, 0.5091114036209083, 0.5450578599083878, 0.5437232517489079, 0.5034351618175156, 0.5173076228323285, 1.3237975068923464, 1.3288440675518258, 1.3248895211467493, 1.3241782160754627, 2.1143087668589984, 2.2312871682409448, 2.2963419698621235, 2.179530309079723, 1.935901488532883, 1.8140845725947157, 1.6046775137322655, 1.5564323816789656, 0.9821558132941463, 0.8522601786353591, 0.771975946680249, 0.7658863461227908 ], "peak_layer": 23, "peak_normalized": 2.2963419698621235, "population_cv": 0.6534359520074113, "rest_mean": 7.539206938047385e-05, "spike_contrast": 2.734585857804243, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020616608671844005, "top_five_layers": [ 23, 22, 24, 21, 25 ], "top_five_spike_overlap": 5 }, "values": [ 8.884922135621309e-05, 3.568552347132936e-05, 1.3508916708815377e-05, 4.1223582229577005e-05, 2.250411307613831e-05, 1.9259987311670557e-05, 2.388052234891802e-05, 4.36920381616801e-05, 7.567495049443096e-05, 5.747437535319477e-05, 5.496075391420163e-05, 4.878585968981497e-05, 5.2230447181500494e-05, 5.210255767451599e-05, 4.824193092645146e-05, 4.957126657245681e-05, 0.00012685357069130987, 0.00012733715993817896, 0.00012695821351371706, 0.00012689005234278738, 0.00020260471501387656, 0.00021381422993727028, 0.0002200481394538656, 0.0002088546025333926, 0.00018550874665379524, 0.0001738355786073953, 0.00015376904048025608, 0.0001491459261160344, 9.41155813052319e-05, 8.166826592059806e-05, 7.397498848149553e-05, 7.33914494048804e-05 ] } } } } } }, "step": 2000 }, { "modes": { "learned": { "bits_per_byte": 1.9147359946947653, "forward": { "activation_sha256": { "attention_branch_output": "729d18a894ede5429f33682ebed5494f0e192bf108efbc8a123308e96fadab4d", "mlp_branch_output": "0c4b69b5a62d474637d9a6156b8452d836a142d489dd60b2de0cdb09db585af9", "post_attention_state": "4cec366fa91114ffbf2b3855220297ceecccb362041740eb1c3b94ecdc658134", "post_mlp_state": "4fbc436930b501f120196fb3bf6ab490fc20b93e2a3a7db48af6d653a1fa07c9", "pre_attention_input": "e4d8ebf5b6b3ad968bf338e0327e2eabdbae498ab18a1157cbd942424ccea56f", "pre_mlp_input": "6920ec3aaf9b121de9e3648519271c2cd5d5a0dcd243425164f6c664720841a8" }, "logits_sha256": "2ffb0289fec25aa61039516a8b1b69c57c89a6b65a12db0380cd5800a2c4f358", "loss_nats": 1.3271938562393188, "mixer_summary_sha256": "8d41a981bb22af8def07f1f5f21c359a1017aaf93614b906571e789e41229366" }, "loss_nats": 1.3271938562393188, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22879935801029205, "group": 1, "latest_source_mass": 0.06730473041534424, "layer": 1, "max_source_mass": 0.9326952695846558, "mixer_index": 1, "normalized_entropy": 0.3300876991600269, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9326952695846558, "median": 0.9494354128837585, "p05": 0.83271324634552, "p95": 0.980820894241333 }, { "label": "current_group_partial", "mean": 0.06730473041534424, "median": 0.05056454986333847, "p05": 0.019179098308086395, "p95": 0.1672867089509964 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5610548257827759, "group": 1, "latest_source_mass": 0.3240153193473816, "layer": 2, "max_source_mass": 0.6759846806526184, "mixer_index": 2, "normalized_entropy": 0.8094310148236321, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.6759846806526184, "median": 0.6873152256011963, "p05": 0.3489363193511963, "p95": 0.9112191796302795 }, { "label": "current_group_partial", "mean": 0.3240153193473816, "median": 0.3126847445964813, "p05": 0.08878080546855927, "p95": 0.6510636806488037 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.43457120656967163, "group": 1, "latest_source_mass": 0.1704302430152893, "layer": 2, "max_source_mass": 0.8295696973800659, "mixer_index": 3, "normalized_entropy": 0.6269537246311986, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8295696973800659, "median": 0.8586000800132751, "p05": 0.6682394742965698, "p95": 0.9325915575027466 }, { "label": "current_group_partial", "mean": 0.1704302430152893, "median": 0.14139993488788605, "p05": 0.06740844249725342, "p95": 0.3317604660987854 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6696063280105591, "group": 1, "latest_source_mass": 0.45400765538215637, "layer": 3, "max_source_mass": 0.545992374420166, "mixer_index": 4, "normalized_entropy": 0.9660377287687022, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.545992374420166, "median": 0.5355842709541321, "p05": 0.40380191802978516, "p95": 0.7150261998176575 }, { "label": "current_group_partial", "mean": 0.45400765538215637, "median": 0.4644157290458679, "p05": 0.2849738299846649, "p95": 0.5961980819702148 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6690812110900879, "group": 1, "latest_source_mass": 0.479726105928421, "layer": 3, "max_source_mass": 0.5202739238739014, "mixer_index": 5, "normalized_entropy": 0.9652801451916515, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5202739238739014, "median": 0.5058591365814209, "p05": 0.3796083331108093, "p95": 0.7244806289672852 }, { "label": "current_group_partial", "mean": 0.479726105928421, "median": 0.4941408932209015, "p05": 0.2755194306373596, "p95": 0.6203916072845459 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6865538358688354, "group": 1, "latest_source_mass": 0.4638097584247589, "layer": 4, "max_source_mass": 0.5361902713775635, "mixer_index": 6, "normalized_entropy": 0.9904878143112643, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5361902713775635, "median": 0.5348554849624634, "p05": 0.4683650732040405, "p95": 0.6130049228668213 }, { "label": "current_group_partial", "mean": 0.4638097584247589, "median": 0.4651445150375366, "p05": 0.3869951069355011, "p95": 0.5316349267959595 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4152516722679138, "group": 1, "latest_source_mass": 0.15472738444805145, "layer": 4, "max_source_mass": 0.8452726602554321, "mixer_index": 7, "normalized_entropy": 0.5990815283017684, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.8452726602554321, "median": 0.8536937236785889, "p05": 0.7318258285522461, "p95": 0.9432691931724548 }, { "label": "current_group_partial", "mean": 0.15472738444805145, "median": 0.14630627632141113, "p05": 0.05673077702522278, "p95": 0.26817411184310913 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6853918433189392, "group": 2, "latest_source_mass": 0.4779263138771057, "layer": 5, "max_source_mass": 0.5220736861228943, "mixer_index": 8, "normalized_entropy": 0.9888114134219791, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5220736861228943, "median": 0.5176622867584229, "p05": 0.4356726408004761, "p95": 0.6261813640594482 }, { "label": "completed_group_1", "mean": 0.4779263138771057, "median": 0.4823376536369324, "p05": 0.37381869554519653, "p95": 0.5643272995948792 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0265861749649048, "group": 2, "latest_source_mass": 0.4458175599575043, "layer": 5, "max_source_mass": 0.4458175599575043, "mixer_index": 9, "normalized_entropy": 0.934439005965858, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3228098154067993, "median": 0.3112887740135193, "p05": 0.17013706266880035, "p95": 0.5363445281982422 }, { "label": "completed_group_1", "mean": 0.23137259483337402, "median": 0.23920756578445435, "p05": 0.15646742284297943, "p95": 0.27411726117134094 }, { "label": "current_group_partial", "mean": 0.4458175599575043, "median": 0.46458810567855835, "p05": 0.26785901188850403, "p95": 0.584333062171936 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.066486120223999, "group": 2, "latest_source_mass": 0.44416186213493347, "layer": 6, "max_source_mass": 0.44416186213493347, "mixer_index": 10, "normalized_entropy": 0.9707575012809491, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2909931242465973, "median": 0.28486818075180054, "p05": 0.24587924778461456, "p95": 0.3592495620250702 }, { "label": "completed_group_1", "mean": 0.26484501361846924, "median": 0.2616615891456604, "p05": 0.23822297155857086, "p95": 0.30263856053352356 }, { "label": "current_group_partial", "mean": 0.44416186213493347, "median": 0.4583350419998169, "p05": 0.3442475199699402, "p95": 0.4912717938423157 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.8915231227874756, "group": 2, "latest_source_mass": 0.6131048202514648, "layer": 6, "max_source_mass": 0.6131048202514648, "mixer_index": 11, "normalized_entropy": 0.8114993178060147, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21873247623443604, "median": 0.1719968020915985, "p05": 0.11714419722557068, "p95": 0.4722311198711395 }, { "label": "completed_group_1", "mean": 0.16816270351409912, "median": 0.16899234056472778, "p05": 0.11237625032663345, "p95": 0.20729371905326843 }, { "label": "current_group_partial", "mean": 0.6131048202514648, "median": 0.6546929478645325, "p05": 0.37714827060699463, "p95": 0.7158147096633911 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0651111602783203, "group": 2, "latest_source_mass": 0.41314971446990967, "layer": 7, "max_source_mass": 0.41314971446990967, "mixer_index": 12, "normalized_entropy": 0.9695059588033517, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.363992840051651, "median": 0.3703998625278473, "p05": 0.312745600938797, "p95": 0.4009946286678314 }, { "label": "completed_group_1", "mean": 0.22285741567611694, "median": 0.219387024641037, "p05": 0.19873447716236115, "p95": 0.2561606168746948 }, { "label": "current_group_partial", "mean": 0.41314971446990967, "median": 0.412320613861084, "p05": 0.37329763174057007, "p95": 0.450311541557312 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0005524158477783, "group": 2, "latest_source_mass": 0.49950775504112244, "layer": 7, "max_source_mass": 0.49950775504112244, "mixer_index": 13, "normalized_entropy": 0.9107420572008955, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.31661832332611084, "median": 0.29119664430618286, "p05": 0.20734748244285583, "p95": 0.5084977149963379 }, { "label": "completed_group_1", "mean": 0.18387392163276672, "median": 0.18630169332027435, "p05": 0.13286109268665314, "p95": 0.22609204053878784 }, { "label": "current_group_partial", "mean": 0.49950775504112244, "median": 0.5250610113143921, "p05": 0.3313663601875305, "p95": 0.5866644382476807 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0648009777069092, "group": 2, "latest_source_mass": 0.41697072982788086, "layer": 8, "max_source_mass": 0.41697072982788086, "mixer_index": 14, "normalized_entropy": 0.9692236184594373, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.35599708557128906, "median": 0.3621290922164917, "p05": 0.286081463098526, "p95": 0.4091617166996002 }, { "label": "completed_group_1", "mean": 0.2270321547985077, "median": 0.2260679304599762, "p05": 0.19767868518829346, "p95": 0.2600743770599365 }, { "label": "current_group_partial", "mean": 0.41697072982788086, "median": 0.41424113512039185, "p05": 0.3673546314239502, "p95": 0.4745914936065674 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0006463527679443, "group": 2, "latest_source_mass": 0.5162267684936523, "layer": 8, "max_source_mass": 0.5162267684936523, "mixer_index": 15, "normalized_entropy": 0.9108275622704591, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28650256991386414, "median": 0.265026330947876, "p05": 0.18542741239070892, "p95": 0.454831063747406 }, { "label": "completed_group_1", "mean": 0.19727063179016113, "median": 0.1941412389278412, "p05": 0.14316384494304657, "p95": 0.2586134374141693 }, { "label": "current_group_partial", "mean": 0.5162267684936523, "median": 0.5309736728668213, "p05": 0.40128540992736816, "p95": 0.5870281457901001 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.072684407234192, "group": 3, "latest_source_mass": 0.34105706214904785, "layer": 9, "max_source_mass": 0.41332393884658813, "mixer_index": 16, "normalized_entropy": 0.9763994252555183, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.41332393884658813, "median": 0.4128713607788086, "p05": 0.3400208353996277, "p95": 0.48814913630485535 }, { "label": "completed_group_1", "mean": 0.2456190288066864, "median": 0.24588967859745026, "p05": 0.21115674078464508, "p95": 0.2809695601463318 }, { "label": "completed_group_2", "mean": 0.34105706214904785, "median": 0.3410349190235138, "p05": 0.29047802090644836, "p95": 0.392678439617157 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.304518699645996, "group": 3, "latest_source_mass": 0.35304954648017883, "layer": 9, "max_source_mass": 0.35304954648017883, "mixer_index": 17, "normalized_entropy": 0.9410113293630988, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2721603810787201, "median": 0.23975250124931335, "p05": 0.16686967015266418, "p95": 0.4960964620113373 }, { "label": "completed_group_1", "mean": 0.12663133442401886, "median": 0.12792861461639404, "p05": 0.07451300323009491, "p95": 0.16275638341903687 }, { "label": "completed_group_2", "mean": 0.2481587529182434, "median": 0.2613898515701294, "p05": 0.14526164531707764, "p95": 0.30340635776519775 }, { "label": "current_group_partial", "mean": 0.35304954648017883, "median": 0.35470759868621826, "p05": 0.26913243532180786, "p95": 0.42660659551620483 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3646502494812012, "group": 3, "latest_source_mass": 0.2896400988101959, "layer": 10, "max_source_mass": 0.29890745878219604, "mixer_index": 18, "normalized_entropy": 0.9843870737372079, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.29890745878219604, "median": 0.29588401317596436, "p05": 0.25596654415130615, "p95": 0.35064268112182617 }, { "label": "completed_group_1", "mean": 0.18950368463993073, "median": 0.18751488626003265, "p05": 0.1636066436767578, "p95": 0.2226080447435379 }, { "label": "completed_group_2", "mean": 0.2219487726688385, "median": 0.22622263431549072, "p05": 0.17594511806964874, "p95": 0.2489231377840042 }, { "label": "current_group_partial", "mean": 0.2896400988101959, "median": 0.2882089614868164, "p05": 0.2503519654273987, "p95": 0.33445414900779724 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.331923484802246, "group": 3, "latest_source_mass": 0.3192984461784363, "layer": 10, "max_source_mass": 0.3192984461784363, "mixer_index": 19, "normalized_entropy": 0.9607797031838735, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.31035515666007996, "median": 0.31498879194259644, "p05": 0.23021729290485382, "p95": 0.40513044595718384 }, { "label": "completed_group_1", "mean": 0.14580611884593964, "median": 0.14570140838623047, "p05": 0.11249937862157822, "p95": 0.1775612086057663 }, { "label": "completed_group_2", "mean": 0.22454024851322174, "median": 0.22962218523025513, "p05": 0.15898653864860535, "p95": 0.2764281928539276 }, { "label": "current_group_partial", "mean": 0.3192984461784363, "median": 0.31587594747543335, "p05": 0.2565876841545105, "p95": 0.3967522978782654 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3639233112335205, "group": 3, "latest_source_mass": 0.29643675684928894, "layer": 11, "max_source_mass": 0.29643675684928894, "mixer_index": 20, "normalized_entropy": 0.9838626986347272, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2952677607536316, "median": 0.2923831343650818, "p05": 0.24788890779018402, "p95": 0.35271933674812317 }, { "label": "completed_group_1", "mean": 0.19548878073692322, "median": 0.19292637705802917, "p05": 0.1706344485282898, "p95": 0.22774341702461243 }, { "label": "completed_group_2", "mean": 0.21280676126480103, "median": 0.21625755727291107, "p05": 0.1678445041179657, "p95": 0.24309992790222168 }, { "label": "current_group_partial", "mean": 0.29643675684928894, "median": 0.2966676950454712, "p05": 0.25015711784362793, "p95": 0.34584304690361023 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2764989137649536, "group": 3, "latest_source_mass": 0.3178994059562683, "layer": 11, "max_source_mass": 0.3656603693962097, "mixer_index": 21, "normalized_entropy": 0.9207993262944236, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3656603693962097, "median": 0.3675394058227539, "p05": 0.19079747796058655, "p95": 0.5280343294143677 }, { "label": "completed_group_1", "mean": 0.14041098952293396, "median": 0.1384333223104477, "p05": 0.09619437158107758, "p95": 0.18906019628047943 }, { "label": "completed_group_2", "mean": 0.17602922022342682, "median": 0.17342498898506165, "p05": 0.10058781504631042, "p95": 0.2688048481941223 }, { "label": "current_group_partial", "mean": 0.3178994059562683, "median": 0.3060161769390106, "p05": 0.2126653641462326, "p95": 0.4575258493423462 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.364103078842163, "group": 3, "latest_source_mass": 0.30165016651153564, "layer": 12, "max_source_mass": 0.30165016651153564, "mixer_index": 22, "normalized_entropy": 0.9839923735534777, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27470695972442627, "median": 0.28049176931381226, "p05": 0.20271319150924683, "p95": 0.3434835374355316 }, { "label": "completed_group_1", "mean": 0.18989428877830505, "median": 0.18842613697052002, "p05": 0.1683974415063858, "p95": 0.21698811650276184 }, { "label": "completed_group_2", "mean": 0.23374855518341064, "median": 0.23892146348953247, "p05": 0.17832231521606445, "p95": 0.2664119601249695 }, { "label": "current_group_partial", "mean": 0.30165016651153564, "median": 0.29482388496398926, "p05": 0.25988030433654785, "p95": 0.36617302894592285 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3429455757141113, "group": 3, "latest_source_mass": 0.3075411319732666, "layer": 12, "max_source_mass": 0.3075411319732666, "mixer_index": 23, "normalized_entropy": 0.9687304611332612, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2830207943916321, "median": 0.28728431463241577, "p05": 0.15201547741889954, "p95": 0.4170074760913849 }, { "label": "completed_group_1", "mean": 0.17677830159664154, "median": 0.17705857753753662, "p05": 0.1337193250656128, "p95": 0.21641075611114502 }, { "label": "completed_group_2", "mean": 0.23265975713729858, "median": 0.2354048639535904, "p05": 0.1727992296218872, "p95": 0.2813771963119507 }, { "label": "current_group_partial", "mean": 0.3075411319732666, "median": 0.29946592450141907, "p05": 0.2323879599571228, "p95": 0.41285809874534607 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3732160329818726, "group": 4, "latest_source_mass": 0.2892792224884033, "layer": 13, "max_source_mass": 0.2892792224884033, "mixer_index": 24, "normalized_entropy": 0.9905659804260815, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2642192244529724, "median": 0.26541125774383545, "p05": 0.22824814915657043, "p95": 0.2958231270313263 }, { "label": "completed_group_1", "mean": 0.22012627124786377, "median": 0.21323975920677185, "p05": 0.19119325280189514, "p95": 0.27720844745635986 }, { "label": "completed_group_2", "mean": 0.22637531161308289, "median": 0.2285899519920349, "p05": 0.193231463432312, "p95": 0.2564765214920044 }, { "label": "completed_group_3", "mean": 0.2892792224884033, "median": 0.2939226031303406, "p05": 0.21020042896270752, "p95": 0.35712480545043945 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5599325895309448, "group": 4, "latest_source_mass": 0.2374764084815979, "layer": 13, "max_source_mass": 0.26679670810699463, "mixer_index": 25, "normalized_entropy": 0.9692406134336154, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.18314200639724731, "median": 0.17654985189437866, "p05": 0.09326011687517166, "p95": 0.32041969895362854 }, { "label": "completed_group_1", "mean": 0.13039523363113403, "median": 0.1256764680147171, "p05": 0.1007242202758789, "p95": 0.17066726088523865 }, { "label": "completed_group_2", "mean": 0.18218961358070374, "median": 0.18480567634105682, "p05": 0.1226210743188858, "p95": 0.23115605115890503 }, { "label": "completed_group_3", "mean": 0.26679670810699463, "median": 0.2663910388946533, "p05": 0.20957797765731812, "p95": 0.3251228928565979 }, { "label": "current_group_partial", "mean": 0.2374764084815979, "median": 0.23747703433036804, "p05": 0.1834154576063156, "p95": 0.2942982316017151 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5772080421447754, "group": 4, "latest_source_mass": 0.2955484390258789, "layer": 14, "max_source_mass": 0.2955484390258789, "mixer_index": 26, "normalized_entropy": 0.9799744556529175, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19838765263557434, "median": 0.1983858346939087, "p05": 0.1743457317352295, "p95": 0.22720122337341309 }, { "label": "completed_group_1", "mean": 0.18937475979328156, "median": 0.18892334401607513, "p05": 0.17057949304580688, "p95": 0.21014319360256195 }, { "label": "completed_group_2", "mean": 0.15325719118118286, "median": 0.15288054943084717, "p05": 0.13430020213127136, "p95": 0.17301824688911438 }, { "label": "completed_group_3", "mean": 0.16343194246292114, "median": 0.1631144881248474, "p05": 0.14038164913654327, "p95": 0.18891242146492004 }, { "label": "current_group_partial", "mean": 0.2955484390258789, "median": 0.2958310842514038, "p05": 0.2728894054889679, "p95": 0.31609299778938293 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5662866830825806, "group": 4, "latest_source_mass": 0.2633046805858612, "layer": 14, "max_source_mass": 0.2633046805858612, "mixer_index": 27, "normalized_entropy": 0.9731886337347067, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.22973918914794922, "median": 0.22824904322624207, "p05": 0.15654821693897247, "p95": 0.29926687479019165 }, { "label": "completed_group_1", "mean": 0.12454323470592499, "median": 0.12315000593662262, "p05": 0.0935351550579071, "p95": 0.15979833900928497 }, { "label": "completed_group_2", "mean": 0.16623356938362122, "median": 0.1699516773223877, "p05": 0.1137569397687912, "p95": 0.20656129717826843 }, { "label": "completed_group_3", "mean": 0.21617934107780457, "median": 0.21567070484161377, "p05": 0.1733308732509613, "p95": 0.2610541582107544 }, { "label": "current_group_partial", "mean": 0.2633046805858612, "median": 0.2623734474182129, "p05": 0.21259599924087524, "p95": 0.3165385127067566 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5421115159988403, "group": 4, "latest_source_mass": 0.3367220163345337, "layer": 15, "max_source_mass": 0.3367220163345337, "mixer_index": 28, "normalized_entropy": 0.9581677578767632, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2021060585975647, "median": 0.2054135799407959, "p05": 0.16289709508419037, "p95": 0.24026834964752197 }, { "label": "completed_group_1", "mean": 0.1677371859550476, "median": 0.1653897762298584, "p05": 0.14229144155979156, "p95": 0.20148998498916626 }, { "label": "completed_group_2", "mean": 0.11891373246908188, "median": 0.11754471063613892, "p05": 0.10472346842288971, "p95": 0.13705110549926758 }, { "label": "completed_group_3", "mean": 0.17452099919319153, "median": 0.17379853129386902, "p05": 0.1422983705997467, "p95": 0.21011367440223694 }, { "label": "current_group_partial", "mean": 0.3367220163345337, "median": 0.33760347962379456, "p05": 0.29649966955184937, "p95": 0.3747695982456207 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.564535140991211, "group": 4, "latest_source_mass": 0.25831347703933716, "layer": 15, "max_source_mass": 0.25831347703933716, "mixer_index": 29, "normalized_entropy": 0.9721003394439871, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.22419840097427368, "median": 0.20440472662448883, "p05": 0.14446967840194702, "p95": 0.3542698621749878 }, { "label": "completed_group_1", "mean": 0.12987476587295532, "median": 0.12998200953006744, "p05": 0.10328994691371918, "p95": 0.15754634141921997 }, { "label": "completed_group_2", "mean": 0.1717071831226349, "median": 0.17557166516780853, "p05": 0.12064125388860703, "p95": 0.21374036371707916 }, { "label": "completed_group_3", "mean": 0.21590615808963776, "median": 0.2161775827407837, "p05": 0.1625441014766693, "p95": 0.2702201008796692 }, { "label": "current_group_partial", "mean": 0.25831347703933716, "median": 0.2593906819820404, "p05": 0.1950702667236328, "p95": 0.3162909150123596 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5267750024795532, "group": 4, "latest_source_mass": 0.33957332372665405, "layer": 16, "max_source_mass": 0.33957332372665405, "mixer_index": 30, "normalized_entropy": 0.9486386462528844, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.215010404586792, "median": 0.22210124135017395, "p05": 0.1559298038482666, "p95": 0.2592989504337311 }, { "label": "completed_group_1", "mean": 0.19819970428943634, "median": 0.19619059562683105, "p05": 0.1762683242559433, "p95": 0.22793664038181305 }, { "label": "completed_group_2", "mean": 0.10917141288518906, "median": 0.10697916150093079, "p05": 0.09485334903001785, "p95": 0.13162456452846527 }, { "label": "completed_group_3", "mean": 0.13804511725902557, "median": 0.13598395884037018, "p05": 0.1083223894238472, "p95": 0.17449748516082764 }, { "label": "current_group_partial", "mean": 0.33957332372665405, "median": 0.3368798792362213, "p05": 0.2983384430408478, "p95": 0.3911387622356415 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.556459903717041, "group": 4, "latest_source_mass": 0.2654845416545868, "layer": 16, "max_source_mass": 0.2654845416545868, "mixer_index": 31, "normalized_entropy": 0.9670829124206874, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2037975788116455, "median": 0.1888633668422699, "p05": 0.12331099063158035, "p95": 0.3977951407432556 }, { "label": "completed_group_1", "mean": 0.12750470638275146, "median": 0.12862002849578857, "p05": 0.09542393684387207, "p95": 0.1604449599981308 }, { "label": "completed_group_2", "mean": 0.1812792420387268, "median": 0.18877843022346497, "p05": 0.11129779368638992, "p95": 0.22781090438365936 }, { "label": "completed_group_3", "mean": 0.22193393111228943, "median": 0.22035372257232666, "p05": 0.1620493084192276, "p95": 0.2887580990791321 }, { "label": "current_group_partial", "mean": 0.2654845416545868, "median": 0.2675127387046814, "p05": 0.18758812546730042, "p95": 0.33534300327301025 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5851528644561768, "group": 5, "latest_source_mass": 0.26602527499198914, "layer": 17, "max_source_mass": 0.26602527499198914, "mixer_index": 32, "normalized_entropy": 0.9849108513038598, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21840544044971466, "median": 0.2208200842142105, "p05": 0.17802955210208893, "p95": 0.2532227635383606 }, { "label": "completed_group_1", "mean": 0.1923592984676361, "median": 0.19034898281097412, "p05": 0.17059199512004852, "p95": 0.22078171372413635 }, { "label": "completed_group_2", "mean": 0.1442098617553711, "median": 0.14471027255058289, "p05": 0.13257130980491638, "p95": 0.15438252687454224 }, { "label": "completed_group_3", "mean": 0.1790001392364502, "median": 0.17785562574863434, "p05": 0.15098975598812103, "p95": 0.2121024876832962 }, { "label": "completed_group_4", "mean": 0.26602527499198914, "median": 0.26503798365592957, "p05": 0.23478449881076813, "p95": 0.2991819977760315 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7158764600753784, "group": 5, "latest_source_mass": 0.3043471872806549, "layer": 17, "max_source_mass": 0.3043471872806549, "mixer_index": 33, "normalized_entropy": 0.9576488862172057, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13390132784843445, "median": 0.12297401577234268, "p05": 0.08219312131404877, "p95": 0.2245325893163681 }, { "label": "completed_group_1", "mean": 0.10310791432857513, "median": 0.10256334394216537, "p05": 0.08573498576879501, "p95": 0.122348852455616 }, { "label": "completed_group_2", "mean": 0.14190615713596344, "median": 0.1444874107837677, "p05": 0.10590264946222305, "p95": 0.16855967044830322 }, { "label": "completed_group_3", "mean": 0.16341811418533325, "median": 0.16260069608688354, "p05": 0.13960278034210205, "p95": 0.18985378742218018 }, { "label": "completed_group_4", "mean": 0.15331928431987762, "median": 0.15326738357543945, "p05": 0.12543924152851105, "p95": 0.18207281827926636 }, { "label": "current_group_partial", "mean": 0.3043471872806549, "median": 0.3056420385837555, "p05": 0.2622474730014801, "p95": 0.34185561537742615 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6410574913024902, "group": 5, "latest_source_mass": 0.3788188695907593, "layer": 18, "max_source_mass": 0.3788188695907593, "mixer_index": 34, "normalized_entropy": 0.9158916246774509, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14665581285953522, "median": 0.15054050087928772, "p05": 0.10432849824428558, "p95": 0.18623962998390198 }, { "label": "completed_group_1", "mean": 0.12231245636940002, "median": 0.12052746117115021, "p05": 0.10417830944061279, "p95": 0.14690443873405457 }, { "label": "completed_group_2", "mean": 0.08100371807813644, "median": 0.08060519397258759, "p05": 0.07169733196496964, "p95": 0.09173959493637085 }, { "label": "completed_group_3", "mean": 0.11105544120073318, "median": 0.109858438372612, "p05": 0.09216424077749252, "p95": 0.13396628201007843 }, { "label": "completed_group_4", "mean": 0.16015371680259705, "median": 0.16027043759822845, "p05": 0.13385453820228577, "p95": 0.18605849146842957 }, { "label": "current_group_partial", "mean": 0.3788188695907593, "median": 0.3796834945678711, "p05": 0.325242280960083, "p95": 0.43094635009765625 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6537573337554932, "group": 5, "latest_source_mass": 0.363794207572937, "layer": 18, "max_source_mass": 0.363794207572937, "mixer_index": 35, "normalized_entropy": 0.9229795417059984, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12217442691326141, "median": 0.11129313707351685, "p05": 0.06773082911968231, "p95": 0.23439142107963562 }, { "label": "completed_group_1", "mean": 0.09171080589294434, "median": 0.09032920002937317, "p05": 0.07566524296998978, "p95": 0.11195167899131775 }, { "label": "completed_group_2", "mean": 0.1348741054534912, "median": 0.13740724325180054, "p05": 0.09137580543756485, "p95": 0.16799552738666534 }, { "label": "completed_group_3", "mean": 0.16064059734344482, "median": 0.15993739664554596, "p05": 0.12922504544258118, "p95": 0.1927689015865326 }, { "label": "completed_group_4", "mean": 0.1268058568239212, "median": 0.1260996162891388, "p05": 0.09770616143941879, "p95": 0.1587117612361908 }, { "label": "current_group_partial", "mean": 0.363794207572937, "median": 0.36618149280548096, "p05": 0.2940343916416168, "p95": 0.4259193539619446 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6571812629699707, "group": 5, "latest_source_mass": 0.3666725754737854, "layer": 19, "max_source_mass": 0.3666725754737854, "mixer_index": 36, "normalized_entropy": 0.9248904729851576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15173450112342834, "median": 0.15636742115020752, "p05": 0.0919112116098404, "p95": 0.20387139916419983 }, { "label": "completed_group_1", "mean": 0.14253422617912292, "median": 0.1411309391260147, "p05": 0.12321583926677704, "p95": 0.16562411189079285 }, { "label": "completed_group_2", "mean": 0.09320533275604248, "median": 0.09343434870243073, "p05": 0.08043995499610901, "p95": 0.10511597990989685 }, { "label": "completed_group_3", "mean": 0.10436245799064636, "median": 0.10301217436790466, "p05": 0.08662116527557373, "p95": 0.1267353594303131 }, { "label": "completed_group_4", "mean": 0.1414909064769745, "median": 0.1412898600101471, "p05": 0.11758942902088165, "p95": 0.1667475551366806 }, { "label": "current_group_partial", "mean": 0.3666725754737854, "median": 0.3659632205963135, "p05": 0.30865201354026794, "p95": 0.429829478263855 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6046829223632812, "group": 5, "latest_source_mass": 0.40004032850265503, "layer": 19, "max_source_mass": 0.40004032850265503, "mixer_index": 37, "normalized_entropy": 0.8955905912162574, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12253663688898087, "median": 0.10175791382789612, "p05": 0.06238773465156555, "p95": 0.2659108340740204 }, { "label": "completed_group_1", "mean": 0.08343374729156494, "median": 0.08214844763278961, "p05": 0.06734234094619751, "p95": 0.10468925535678864 }, { "label": "completed_group_2", "mean": 0.1302124410867691, "median": 0.13176652789115906, "p05": 0.08769658207893372, "p95": 0.167977973818779 }, { "label": "completed_group_3", "mean": 0.14439499378204346, "median": 0.1426759511232376, "p05": 0.10537415742874146, "p95": 0.188823863863945 }, { "label": "completed_group_4", "mean": 0.11938183009624481, "median": 0.11938531696796417, "p05": 0.08638042211532593, "p95": 0.15211962163448334 }, { "label": "current_group_partial", "mean": 0.40004032850265503, "median": 0.40058958530426025, "p05": 0.30273354053497314, "p95": 0.49229907989501953 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7041404247283936, "group": 5, "latest_source_mass": 0.3168083429336548, "layer": 20, "max_source_mass": 0.3168083429336548, "mixer_index": 38, "normalized_entropy": 0.9510988801764724, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1659325659275055, "median": 0.17128247022628784, "p05": 0.10529995709657669, "p95": 0.22297045588493347 }, { "label": "completed_group_1", "mean": 0.15505464375019073, "median": 0.15309038758277893, "p05": 0.1348525583744049, "p95": 0.18119457364082336 }, { "label": "completed_group_2", "mean": 0.10219550132751465, "median": 0.10201787948608398, "p05": 0.08870653063058853, "p95": 0.11639079451560974 }, { "label": "completed_group_3", "mean": 0.11347829550504684, "median": 0.11260586977005005, "p05": 0.09271398186683655, "p95": 0.13755615055561066 }, { "label": "completed_group_4", "mean": 0.1465306580066681, "median": 0.14615076780319214, "p05": 0.1223587766289711, "p95": 0.17159976065158844 }, { "label": "current_group_partial", "mean": 0.3168083429336548, "median": 0.31445345282554626, "p05": 0.2601228952407837, "p95": 0.3804415166378021 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5808616876602173, "group": 5, "latest_source_mass": 0.40230536460876465, "layer": 20, "max_source_mass": 0.40230536460876465, "mixer_index": 39, "normalized_entropy": 0.882295706990906, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12713032960891724, "median": 0.10352589935064316, "p05": 0.03667577728629112, "p95": 0.34593111276626587 }, { "label": "completed_group_1", "mean": 0.08526259660720825, "median": 0.08378827571868896, "p05": 0.0644085630774498, "p95": 0.11194434762001038 }, { "label": "completed_group_2", "mean": 0.1294247955083847, "median": 0.13107044994831085, "p05": 0.08035385608673096, "p95": 0.1728026270866394 }, { "label": "completed_group_3", "mean": 0.1489948183298111, "median": 0.14636963605880737, "p05": 0.10222268849611282, "p95": 0.2024523913860321 }, { "label": "completed_group_4", "mean": 0.10688205808401108, "median": 0.10608744621276855, "p05": 0.0690753310918808, "p95": 0.14802516996860504 }, { "label": "current_group_partial", "mean": 0.40230536460876465, "median": 0.4073683023452759, "p05": 0.26591286063194275, "p95": 0.5286550521850586 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7305450439453125, "group": 6, "latest_source_mass": 0.25957077741622925, "layer": 21, "max_source_mass": 0.25957077741622925, "mixer_index": 40, "normalized_entropy": 0.9658355787514741, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1789149045944214, "median": 0.18693166971206665, "p05": 0.11472857743501663, "p95": 0.22855065762996674 }, { "label": "completed_group_1", "mean": 0.17673785984516144, "median": 0.17547538876533508, "p05": 0.15558482706546783, "p95": 0.20077811181545258 }, { "label": "completed_group_2", "mean": 0.09075746685266495, "median": 0.090105801820755, "p05": 0.07735756039619446, "p95": 0.10616939514875412 }, { "label": "completed_group_3", "mean": 0.11627361923456192, "median": 0.11497560888528824, "p05": 0.09235620498657227, "p95": 0.14503517746925354 }, { "label": "completed_group_4", "mean": 0.17774535715579987, "median": 0.17699119448661804, "p05": 0.15380802750587463, "p95": 0.2042580246925354 }, { "label": "completed_group_5", "mean": 0.25957077741622925, "median": 0.2581925392150879, "p05": 0.20536908507347107, "p95": 0.31587323546409607 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7513642311096191, "group": 6, "latest_source_mass": 0.3233836889266968, "layer": 21, "max_source_mass": 0.3233836889266968, "mixer_index": 41, "normalized_entropy": 0.9000231752529063, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09819091856479645, "median": 0.08258114755153656, "p05": 0.05219355598092079, "p95": 0.2192530781030655 }, { "label": "completed_group_1", "mean": 0.06520213931798935, "median": 0.06374529749155045, "p05": 0.051295794546604156, "p95": 0.08378917723894119 }, { "label": "completed_group_2", "mean": 0.091708242893219, "median": 0.09206989407539368, "p05": 0.06343865394592285, "p95": 0.11800471693277359 }, { "label": "completed_group_3", "mean": 0.11103156208992004, "median": 0.10943961143493652, "p05": 0.0842345654964447, "p95": 0.14296333491802216 }, { "label": "completed_group_4", "mean": 0.08388154208660126, "median": 0.08271945267915726, "p05": 0.06058824062347412, "p95": 0.11240207403898239 }, { "label": "completed_group_5", "mean": 0.22660189867019653, "median": 0.22274313867092133, "p05": 0.14589408040046692, "p95": 0.32426023483276367 }, { "label": "current_group_partial", "mean": 0.3233836889266968, "median": 0.32124656438827515, "p05": 0.23837368190288544, "p95": 0.41599756479263306 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.766061782836914, "group": 6, "latest_source_mass": 0.36142250895500183, "layer": 22, "max_source_mass": 0.36142250895500183, "mixer_index": 42, "normalized_entropy": 0.9075762227224567, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12103402614593506, "median": 0.12304593622684479, "p05": 0.07208628952503204, "p95": 0.17526710033416748 }, { "label": "completed_group_1", "mean": 0.1159343421459198, "median": 0.11432909965515137, "p05": 0.09892642498016357, "p95": 0.13806137442588806 }, { "label": "completed_group_2", "mean": 0.06956179440021515, "median": 0.06937768310308456, "p05": 0.059766486287117004, "p95": 0.08032344281673431 }, { "label": "completed_group_3", "mean": 0.07680170238018036, "median": 0.07588358968496323, "p05": 0.06175529211759567, "p95": 0.09538999199867249 }, { "label": "completed_group_4", "mean": 0.10650800168514252, "median": 0.10561707615852356, "p05": 0.08491266518831253, "p95": 0.13118049502372742 }, { "label": "completed_group_5", "mean": 0.1487376093864441, "median": 0.1469218134880066, "p05": 0.11073103547096252, "p95": 0.19332705438137054 }, { "label": "current_group_partial", "mean": 0.36142250895500183, "median": 0.36172276735305786, "p05": 0.2831430435180664, "p95": 0.43728554248809814 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6586570739746094, "group": 6, "latest_source_mass": 0.4119114875793457, "layer": 22, "max_source_mass": 0.4119114875793457, "mixer_index": 43, "normalized_entropy": 0.8523811208754127, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09605097770690918, "median": 0.08114366233348846, "p05": 0.047440964728593826, "p95": 0.2204016000032425 }, { "label": "completed_group_1", "mean": 0.06012085825204849, "median": 0.05865291506052017, "p05": 0.04616369307041168, "p95": 0.07940929383039474 }, { "label": "completed_group_2", "mean": 0.08562932163476944, "median": 0.08538836240768433, "p05": 0.05786459147930145, "p95": 0.11294843256473541 }, { "label": "completed_group_3", "mean": 0.10102558135986328, "median": 0.09871675074100494, "p05": 0.07363428175449371, "p95": 0.1344296932220459 }, { "label": "completed_group_4", "mean": 0.07161714881658554, "median": 0.07036904245615005, "p05": 0.049177274107933044, "p95": 0.09921920299530029 }, { "label": "completed_group_5", "mean": 0.17364460229873657, "median": 0.16451683640480042, "p05": 0.09478501230478287, "p95": 0.2832472622394562 }, { "label": "current_group_partial", "mean": 0.4119114875793457, "median": 0.4100137948989868, "p05": 0.275837242603302, "p95": 0.5469895601272583 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7821578979492188, "group": 6, "latest_source_mass": 0.3514069616794586, "layer": 23, "max_source_mass": 0.3514069616794586, "mixer_index": 44, "normalized_entropy": 0.9158479895972629, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13399384915828705, "median": 0.1371074914932251, "p05": 0.08264479786157608, "p95": 0.1757817566394806 }, { "label": "completed_group_1", "mean": 0.12986589968204498, "median": 0.12847267091274261, "p05": 0.1112535148859024, "p95": 0.1538200080394745 }, { "label": "completed_group_2", "mean": 0.0802837461233139, "median": 0.07979932427406311, "p05": 0.06686097383499146, "p95": 0.09519629925489426 }, { "label": "completed_group_3", "mean": 0.07629825174808502, "median": 0.074957937002182, "p05": 0.060364846140146255, "p95": 0.09640902280807495 }, { "label": "completed_group_4", "mean": 0.10134232044219971, "median": 0.10061150044202805, "p05": 0.0801728367805481, "p95": 0.12531554698944092 }, { "label": "completed_group_5", "mean": 0.12680892646312714, "median": 0.1246437132358551, "p05": 0.09191874414682388, "p95": 0.16937871277332306 }, { "label": "current_group_partial", "mean": 0.3514069616794586, "median": 0.3516124486923218, "p05": 0.2630184292793274, "p95": 0.4369184076786041 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.655133843421936, "group": 6, "latest_source_mass": 0.43040627241134644, "layer": 23, "max_source_mass": 0.43040627241134644, "mixer_index": 45, "normalized_entropy": 0.8505705385346074, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10968074202537537, "median": 0.10342554748058319, "p05": 0.058600373566150665, "p95": 0.19649700820446014 }, { "label": "completed_group_1", "mean": 0.0657510757446289, "median": 0.06393904238939285, "p05": 0.04856989532709122, "p95": 0.08747231960296631 }, { "label": "completed_group_2", "mean": 0.08727139234542847, "median": 0.08742742240428925, "p05": 0.06016327068209648, "p95": 0.11422628164291382 }, { "label": "completed_group_3", "mean": 0.09466101229190826, "median": 0.09291957318782806, "p05": 0.06727220118045807, "p95": 0.12599577009677887 }, { "label": "completed_group_4", "mean": 0.0703636035323143, "median": 0.06896525621414185, "p05": 0.05082611367106438, "p95": 0.0946161150932312 }, { "label": "completed_group_5", "mean": 0.14186589419841766, "median": 0.13478878140449524, "p05": 0.07877384126186371, "p95": 0.2290385663509369 }, { "label": "current_group_partial", "mean": 0.43040627241134644, "median": 0.42919087409973145, "p05": 0.2915465831756592, "p95": 0.5688671469688416 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8160101175308228, "group": 6, "latest_source_mass": 0.3277372717857361, "layer": 24, "max_source_mass": 0.3277372717857361, "mixer_index": 46, "normalized_entropy": 0.933244589125786, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13769462704658508, "median": 0.13593362271785736, "p05": 0.09772415459156036, "p95": 0.18957549333572388 }, { "label": "completed_group_1", "mean": 0.11569900810718536, "median": 0.11375441402196884, "p05": 0.0982208400964737, "p95": 0.1386219710111618 }, { "label": "completed_group_2", "mean": 0.08787183463573456, "median": 0.08727885782718658, "p05": 0.07391972839832306, "p95": 0.10365156829357147 }, { "label": "completed_group_3", "mean": 0.0929793268442154, "median": 0.09170648455619812, "p05": 0.07486406713724136, "p95": 0.11526438593864441 }, { "label": "completed_group_4", "mean": 0.10649851709604263, "median": 0.10617373883724213, "p05": 0.08636443316936493, "p95": 0.12787018716335297 }, { "label": "completed_group_5", "mean": 0.1315194070339203, "median": 0.12966084480285645, "p05": 0.09266573190689087, "p95": 0.17860902845859528 }, { "label": "current_group_partial", "mean": 0.3277372717857361, "median": 0.32655179500579834, "p05": 0.24662655591964722, "p95": 0.4135902523994446 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6999847888946533, "group": 6, "latest_source_mass": 0.3966374695301056, "layer": 24, "max_source_mass": 0.3966374695301056, "mixer_index": 47, "normalized_entropy": 0.873619365066753, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10220035910606384, "median": 0.0844031423330307, "p05": 0.04488028585910797, "p95": 0.24769487977027893 }, { "label": "completed_group_1", "mean": 0.07546016573905945, "median": 0.07393044233322144, "p05": 0.05928082764148712, "p95": 0.09592372179031372 }, { "label": "completed_group_2", "mean": 0.1028895378112793, "median": 0.10241077095270157, "p05": 0.07516536116600037, "p95": 0.13291233777999878 }, { "label": "completed_group_3", "mean": 0.10639216750860214, "median": 0.10462376475334167, "p05": 0.07692230492830276, "p95": 0.14082345366477966 }, { "label": "completed_group_4", "mean": 0.07665040343999863, "median": 0.07448552548885345, "p05": 0.05249025300145149, "p95": 0.10721437633037567 }, { "label": "completed_group_5", "mean": 0.13976991176605225, "median": 0.12892484664916992, "p05": 0.0740278959274292, "p95": 0.24029651284217834 }, { "label": "current_group_partial", "mean": 0.3966374695301056, "median": 0.39967864751815796, "p05": 0.25664323568344116, "p95": 0.525518000125885 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8706753253936768, "group": 7, "latest_source_mass": 0.27128130197525024, "layer": 25, "max_source_mass": 0.27128130197525024, "mixer_index": 48, "normalized_entropy": 0.9613369488318045, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1469482183456421, "median": 0.14645828306674957, "p05": 0.10641424357891083, "p95": 0.1947372704744339 }, { "label": "completed_group_1", "mean": 0.12422746419906616, "median": 0.12275843322277069, "p05": 0.10553248971700668, "p95": 0.14753170311450958 }, { "label": "completed_group_2", "mean": 0.09348422288894653, "median": 0.09236403554677963, "p05": 0.07779940217733383, "p95": 0.11272361129522324 }, { "label": "completed_group_3", "mean": 0.1027352511882782, "median": 0.10125023126602173, "p05": 0.08420244604349136, "p95": 0.12524712085723877 }, { "label": "completed_group_4", "mean": 0.12104703485965729, "median": 0.12076126039028168, "p05": 0.09965822100639343, "p95": 0.14355391263961792 }, { "label": "completed_group_5", "mean": 0.14027650654315948, "median": 0.139203742146492, "p05": 0.10420101135969162, "p95": 0.18390552699565887 }, { "label": "completed_group_6", "mean": 0.27128130197525024, "median": 0.2698158919811249, "p05": 0.20765253901481628, "p95": 0.3398803472518921 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7911043167114258, "group": 7, "latest_source_mass": 0.27496105432510376, "layer": 25, "max_source_mass": 0.27496105432510376, "mixer_index": 49, "normalized_entropy": 0.8613391051447965, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09078208357095718, "median": 0.0916827917098999, "p05": 0.030423305928707123, "p95": 0.16150596737861633 }, { "label": "completed_group_1", "mean": 0.05932023376226425, "median": 0.05782822519540787, "p05": 0.04296479746699333, "p95": 0.08013182133436203 }, { "label": "completed_group_2", "mean": 0.06744871288537979, "median": 0.06634571403265, "p05": 0.04800695925951004, "p95": 0.09060627222061157 }, { "label": "completed_group_3", "mean": 0.0717800185084343, "median": 0.0682687982916832, "p05": 0.04594217985868454, "p95": 0.10765853524208069 }, { "label": "completed_group_4", "mean": 0.05460817739367485, "median": 0.052827149629592896, "p05": 0.03489609435200691, "p95": 0.08040964603424072 }, { "label": "completed_group_5", "mean": 0.10924962162971497, "median": 0.09424155950546265, "p05": 0.047475166618824005, "p95": 0.2151142656803131 }, { "label": "completed_group_6", "mean": 0.2718501091003418, "median": 0.2685811221599579, "p05": 0.14285480976104736, "p95": 0.40964099764823914 }, { "label": "current_group_partial", "mean": 0.27496105432510376, "median": 0.2828528583049774, "p05": 0.06592769920825958, "p95": 0.48006877303123474 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9753689765930176, "group": 7, "latest_source_mass": 0.23982003331184387, "layer": 26, "max_source_mass": 0.23982003331184387, "mixer_index": 50, "normalized_entropy": 0.9499516754855512, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10981251299381256, "median": 0.10871805250644684, "p05": 0.050737276673316956, "p95": 0.1706625372171402 }, { "label": "completed_group_1", "mean": 0.10515634715557098, "median": 0.10322865843772888, "p05": 0.08713022619485855, "p95": 0.12963688373565674 }, { "label": "completed_group_2", "mean": 0.07847881317138672, "median": 0.07656338810920715, "p05": 0.05796374753117561, "p95": 0.10510093718767166 }, { "label": "completed_group_3", "mean": 0.080687016248703, "median": 0.07948547601699829, "p05": 0.06504379212856293, "p95": 0.09981580078601837 }, { "label": "completed_group_4", "mean": 0.10027816891670227, "median": 0.09996622800827026, "p05": 0.07846689224243164, "p95": 0.1242886409163475 }, { "label": "completed_group_5", "mean": 0.1099884957075119, "median": 0.10788669437170029, "p05": 0.07500496506690979, "p95": 0.15274949371814728 }, { "label": "completed_group_6", "mean": 0.1757785975933075, "median": 0.1735863983631134, "p05": 0.12249398976564407, "p95": 0.23682411015033722 }, { "label": "current_group_partial", "mean": 0.23982003331184387, "median": 0.23704850673675537, "p05": 0.11852282285690308, "p95": 0.3622197210788727 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.81705904006958, "group": 7, "latest_source_mass": 0.3062949776649475, "layer": 26, "max_source_mass": 0.3062949776649475, "mixer_index": 51, "normalized_entropy": 0.8738206887036145, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0980612188577652, "median": 0.09061099588871002, "p05": 0.06147613376379013, "p95": 0.1617073118686676 }, { "label": "completed_group_1", "mean": 0.060419049113988876, "median": 0.05925498157739639, "p05": 0.04340171068906784, "p95": 0.08076231181621552 }, { "label": "completed_group_2", "mean": 0.06808654963970184, "median": 0.06729011237621307, "p05": 0.04935084655880928, "p95": 0.08873726427555084 }, { "label": "completed_group_3", "mean": 0.07581339031457901, "median": 0.07308594882488251, "p05": 0.049850281327962875, "p95": 0.10994531959295273 }, { "label": "completed_group_4", "mean": 0.06074263155460358, "median": 0.05964510142803192, "p05": 0.04034031927585602, "p95": 0.08569400757551193 }, { "label": "completed_group_5", "mean": 0.09064903855323792, "median": 0.08298017829656601, "p05": 0.047057293355464935, "p95": 0.15634870529174805 }, { "label": "completed_group_6", "mean": 0.23993313312530518, "median": 0.23315298557281494, "p05": 0.11595168709754944, "p95": 0.3906308114528656 }, { "label": "current_group_partial", "mean": 0.3062949776649475, "median": 0.3047541677951813, "p05": 0.12540288269519806, "p95": 0.4860513210296631 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9814136028289795, "group": 7, "latest_source_mass": 0.23084799945354462, "layer": 27, "max_source_mass": 0.23084799945354462, "mixer_index": 52, "normalized_entropy": 0.9528585262504343, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11045937240123749, "median": 0.10626375675201416, "p05": 0.05494675412774086, "p95": 0.17410020530223846 }, { "label": "completed_group_1", "mean": 0.08757395297288895, "median": 0.08668297529220581, "p05": 0.07028551399707794, "p95": 0.10834703594446182 }, { "label": "completed_group_2", "mean": 0.08714326471090317, "median": 0.0863339900970459, "p05": 0.06873977184295654, "p95": 0.10879559814929962 }, { "label": "completed_group_3", "mean": 0.09700852632522583, "median": 0.0958750918507576, "p05": 0.07455667108297348, "p95": 0.12356264889240265 }, { "label": "completed_group_4", "mean": 0.09435248374938965, "median": 0.09339165687561035, "p05": 0.0721961036324501, "p95": 0.11932304501533508 }, { "label": "completed_group_5", "mean": 0.1148696318268776, "median": 0.11229501664638519, "p05": 0.07511283457279205, "p95": 0.16472017765045166 }, { "label": "completed_group_6", "mean": 0.1777447611093521, "median": 0.17455129325389862, "p05": 0.1177421361207962, "p95": 0.24785079061985016 }, { "label": "current_group_partial", "mean": 0.23084799945354462, "median": 0.2267751693725586, "p05": 0.11723043769598007, "p95": 0.3665117919445038 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8373757600784302, "group": 7, "latest_source_mass": 0.25260502099990845, "layer": 27, "max_source_mass": 0.25260502099990845, "mixer_index": 53, "normalized_entropy": 0.8835909657715805, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08649781346321106, "median": 0.05945904552936554, "p05": 0.026423925533890724, "p95": 0.23923249542713165 }, { "label": "completed_group_1", "mean": 0.07379771023988724, "median": 0.07217878103256226, "p05": 0.04322340711951256, "p95": 0.11003518849611282 }, { "label": "completed_group_2", "mean": 0.0769633874297142, "median": 0.0755230188369751, "p05": 0.04856198653578758, "p95": 0.10847672075033188 }, { "label": "completed_group_3", "mean": 0.0880836620926857, "median": 0.08413928747177124, "p05": 0.04961243271827698, "p95": 0.13771715760231018 }, { "label": "completed_group_4", "mean": 0.066677525639534, "median": 0.0648220106959343, "p05": 0.0391211211681366, "p95": 0.10059694200754166 }, { "label": "completed_group_5", "mean": 0.11875502020120621, "median": 0.10550784319639206, "p05": 0.05534175783395767, "p95": 0.22876456379890442 }, { "label": "completed_group_6", "mean": 0.23661985993385315, "median": 0.22938117384910583, "p05": 0.11185489594936371, "p95": 0.386798620223999 }, { "label": "current_group_partial", "mean": 0.25260502099990845, "median": 0.23003548383712769, "p05": 0.06494839489459991, "p95": 0.5125646591186523 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.968585729598999, "group": 7, "latest_source_mass": 0.22389832139015198, "layer": 28, "max_source_mass": 0.22389832139015198, "mixer_index": 54, "normalized_entropy": 0.946689623219086, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10324695706367493, "median": 0.0941208153963089, "p05": 0.039645131677389145, "p95": 0.1829081028699875 }, { "label": "completed_group_1", "mean": 0.0860632136464119, "median": 0.08438493311405182, "p05": 0.04368945211172104, "p95": 0.14022991061210632 }, { "label": "completed_group_2", "mean": 0.09797552973031998, "median": 0.09780187904834747, "p05": 0.06858807802200317, "p95": 0.1295054405927658 }, { "label": "completed_group_3", "mean": 0.1271316111087799, "median": 0.12579670548439026, "p05": 0.09182935208082199, "p95": 0.16805431246757507 }, { "label": "completed_group_4", "mean": 0.08284065127372742, "median": 0.08147171139717102, "p05": 0.0564071387052536, "p95": 0.11312974244356155 }, { "label": "completed_group_5", "mean": 0.10640716552734375, "median": 0.10340218245983124, "p05": 0.06448834389448166, "p95": 0.15891367197036743 }, { "label": "completed_group_6", "mean": 0.17243653535842896, "median": 0.16770902276039124, "p05": 0.1087951734662056, "p95": 0.24907639622688293 }, { "label": "current_group_partial", "mean": 0.22389832139015198, "median": 0.21250957250595093, "p05": 0.09934276342391968, "p95": 0.3857041895389557 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8529754877090454, "group": 7, "latest_source_mass": 0.20538833737373352, "layer": 28, "max_source_mass": 0.2897399663925171, "mixer_index": 55, "normalized_entropy": 0.8910928490022162, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08385773003101349, "median": 0.06364437937736511, "p05": 0.04014988988637924, "p95": 0.2496711015701294 }, { "label": "completed_group_1", "mean": 0.07188580930233002, "median": 0.07024423032999039, "p05": 0.05124984309077263, "p95": 0.09714871644973755 }, { "label": "completed_group_2", "mean": 0.08174896240234375, "median": 0.07993435859680176, "p05": 0.049406472593545914, "p95": 0.11908631026744843 }, { "label": "completed_group_3", "mean": 0.09558321535587311, "median": 0.0924227386713028, "p05": 0.05770375207066536, "p95": 0.14457376301288605 }, { "label": "completed_group_4", "mean": 0.06307300925254822, "median": 0.06175791472196579, "p05": 0.040213022381067276, "p95": 0.0907377302646637 }, { "label": "completed_group_5", "mean": 0.108722984790802, "median": 0.1007031798362732, "p05": 0.05666481703519821, "p95": 0.1869736760854721 }, { "label": "completed_group_6", "mean": 0.2897399663925171, "median": 0.28985595703125, "p05": 0.14349502325057983, "p95": 0.4364045262336731 }, { "label": "current_group_partial", "mean": 0.20538833737373352, "median": 0.18734587728977203, "p05": 0.07006806135177612, "p95": 0.39856192469596863 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9637342691421509, "group": 8, "latest_source_mass": 0.13441291451454163, "layer": 29, "max_source_mass": 0.2349606156349182, "mixer_index": 56, "normalized_entropy": 0.9443565639050314, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10303820669651031, "median": 0.0819694995880127, "p05": 0.030241798609495163, "p95": 0.2497645914554596 }, { "label": "completed_group_1", "mean": 0.08717728406190872, "median": 0.08418120443820953, "p05": 0.06677442789077759, "p95": 0.11682014167308807 }, { "label": "completed_group_2", "mean": 0.09220535308122635, "median": 0.09127489477396011, "p05": 0.06960588693618774, "p95": 0.11840611696243286 }, { "label": "completed_group_3", "mean": 0.11336663365364075, "median": 0.11189766228199005, "p05": 0.08372759819030762, "p95": 0.1468089371919632 }, { "label": "completed_group_4", "mean": 0.08865514397621155, "median": 0.08804303407669067, "p05": 0.06345562636852264, "p95": 0.11571189761161804 }, { "label": "completed_group_5", "mean": 0.14618384838104248, "median": 0.1397639960050583, "p05": 0.08802400529384613, "p95": 0.22461283206939697 }, { "label": "completed_group_6", "mean": 0.2349606156349182, "median": 0.23364676535129547, "p05": 0.1564730703830719, "p95": 0.317343533039093 }, { "label": "completed_group_7", "mean": 0.13441291451454163, "median": 0.11298182606697083, "p05": 0.049395374953746796, "p95": 0.2780357003211975 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.962258219718933, "group": 8, "latest_source_mass": 0.24145527184009552, "layer": 29, "max_source_mass": 0.24145527184009552, "mixer_index": 57, "normalized_entropy": 0.8930622021795581, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07841142266988754, "median": 0.06509651243686676, "p05": 0.027810024097561836, "p95": 0.17502810060977936 }, { "label": "completed_group_1", "mean": 0.07437640428543091, "median": 0.07133704423904419, "p05": 0.039178602397441864, "p95": 0.12137842923402786 }, { "label": "completed_group_2", "mean": 0.07400830835103989, "median": 0.07284313440322876, "p05": 0.047712553292512894, "p95": 0.10446867346763611 }, { "label": "completed_group_3", "mean": 0.07925957441329956, "median": 0.07685956358909607, "p05": 0.04267336428165436, "p95": 0.12466679513454437 }, { "label": "completed_group_4", "mean": 0.05560165271162987, "median": 0.05358406901359558, "p05": 0.031528957188129425, "p95": 0.08545175939798355 }, { "label": "completed_group_5", "mean": 0.08937598019838333, "median": 0.08107791095972061, "p05": 0.038014721125364304, "p95": 0.1699092984199524 }, { "label": "completed_group_6", "mean": 0.19564677774906158, "median": 0.1907363384962082, "p05": 0.10008887201547623, "p95": 0.3112020492553711 }, { "label": "completed_group_7", "mean": 0.1118645966053009, "median": 0.08986739069223404, "p05": 0.03501077741384506, "p95": 0.26519542932510376 }, { "label": "current_group_partial", "mean": 0.24145527184009552, "median": 0.22476747632026672, "p05": 0.05287282541394234, "p95": 0.49067121744155884 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0993216037750244, "group": 8, "latest_source_mass": 0.12042839825153351, "layer": 30, "max_source_mass": 0.1680138111114502, "mixer_index": 58, "normalized_entropy": 0.955442436530595, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1043272390961647, "median": 0.0907609686255455, "p05": 0.016956502571702003, "p95": 0.284129798412323 }, { "label": "completed_group_1", "mean": 0.09323035180568695, "median": 0.0917273461818695, "p05": 0.068887859582901, "p95": 0.1232273131608963 }, { "label": "completed_group_2", "mean": 0.08778750896453857, "median": 0.08604135364294052, "p05": 0.061383333057165146, "p95": 0.12025082856416702 }, { "label": "completed_group_3", "mean": 0.10311444103717804, "median": 0.10148647427558899, "p05": 0.07428233325481415, "p95": 0.13808897137641907 }, { "label": "completed_group_4", "mean": 0.09694567322731018, "median": 0.0953250527381897, "p05": 0.06249001622200012, "p95": 0.13581177592277527 }, { "label": "completed_group_5", "mean": 0.13869863748550415, "median": 0.1327243149280548, "p05": 0.08499303460121155, "p95": 0.21259371936321259 }, { "label": "completed_group_6", "mean": 0.1680138111114502, "median": 0.16382917761802673, "p05": 0.10980678349733353, "p95": 0.23901790380477905 }, { "label": "completed_group_7", "mean": 0.0874539315700531, "median": 0.07735057175159454, "p05": 0.03232480213046074, "p95": 0.17377637326717377 }, { "label": "current_group_partial", "mean": 0.12042839825153351, "median": 0.10099855065345764, "p05": 0.041284509003162384, "p95": 0.2534906268119812 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0489909648895264, "group": 8, "latest_source_mass": 0.16280405223369598, "layer": 30, "max_source_mass": 0.18801958858966827, "mixer_index": 59, "normalized_entropy": 0.9325359756232098, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0919051468372345, "median": 0.08924420177936554, "p05": 0.04205578938126564, "p95": 0.14709380269050598 }, { "label": "completed_group_1", "mean": 0.07878251373767853, "median": 0.07600399851799011, "p05": 0.04804171249270439, "p95": 0.11802247911691666 }, { "label": "completed_group_2", "mean": 0.07574650645256042, "median": 0.07504475116729736, "p05": 0.051915232092142105, "p95": 0.10225261747837067 }, { "label": "completed_group_3", "mean": 0.09311886876821518, "median": 0.0916299819946289, "p05": 0.05949338898062706, "p95": 0.13063858449459076 }, { "label": "completed_group_4", "mean": 0.05773407965898514, "median": 0.05621109902858734, "p05": 0.03913509100675583, "p95": 0.0801405981183052 }, { "label": "completed_group_5", "mean": 0.07325181365013123, "median": 0.0683174878358841, "p05": 0.039874374866485596, "p95": 0.12237872183322906 }, { "label": "completed_group_6", "mean": 0.17863741517066956, "median": 0.17224830389022827, "p05": 0.09081655740737915, "p95": 0.29137635231018066 }, { "label": "completed_group_7", "mean": 0.18801958858966827, "median": 0.1791573464870453, "p05": 0.08688034117221832, "p95": 0.3170011341571808 }, { "label": "current_group_partial", "mean": 0.16280405223369598, "median": 0.14800652861595154, "p05": 0.058800291270017624, "p95": 0.3165009021759033 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.126072406768799, "group": 8, "latest_source_mass": 0.11243770271539688, "layer": 31, "max_source_mass": 0.17183929681777954, "mixer_index": 60, "normalized_entropy": 0.967617251644945, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09415344893932343, "median": 0.09509380161762238, "p05": 0.032531045377254486, "p95": 0.17404073476791382 }, { "label": "completed_group_1", "mean": 0.07414552569389343, "median": 0.07293231785297394, "p05": 0.043177857995033264, "p95": 0.10848568379878998 }, { "label": "completed_group_2", "mean": 0.10936354845762253, "median": 0.10600043833255768, "p05": 0.07479241490364075, "p95": 0.15432706475257874 }, { "label": "completed_group_3", "mean": 0.11765885353088379, "median": 0.11672843247652054, "p05": 0.0860721692442894, "p95": 0.1508050411939621 }, { "label": "completed_group_4", "mean": 0.07923046499490738, "median": 0.07799427211284637, "p05": 0.05631105974316597, "p95": 0.1065656989812851 }, { "label": "completed_group_5", "mean": 0.12059831619262695, "median": 0.11797575652599335, "p05": 0.07954882085323334, "p95": 0.17233142256736755 }, { "label": "completed_group_6", "mean": 0.17183929681777954, "median": 0.16947603225708008, "p05": 0.11957718431949615, "p95": 0.23197905719280243 }, { "label": "completed_group_7", "mean": 0.12057285010814667, "median": 0.11689606308937073, "p05": 0.058023348450660706, "p95": 0.2011682540178299 }, { "label": "current_group_partial", "mean": 0.11243770271539688, "median": 0.10209996998310089, "p05": 0.057224418967962265, "p95": 0.2040310949087143 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.028747081756592, "group": 8, "latest_source_mass": 0.11574049293994904, "layer": 31, "max_source_mass": 0.24226921796798706, "mixer_index": 61, "normalized_entropy": 0.9233225873597866, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08796507120132446, "median": 0.07151719927787781, "p05": 0.048716649413108826, "p95": 0.21155157685279846 }, { "label": "completed_group_1", "mean": 0.0757753923535347, "median": 0.07367455959320068, "p05": 0.05229219049215317, "p95": 0.10657010972499847 }, { "label": "completed_group_2", "mean": 0.0847562700510025, "median": 0.08441580832004547, "p05": 0.0502907857298851, "p95": 0.12154337763786316 }, { "label": "completed_group_3", "mean": 0.09340266138315201, "median": 0.09229434281587601, "p05": 0.05622609704732895, "p95": 0.13404318690299988 }, { "label": "completed_group_4", "mean": 0.06450396031141281, "median": 0.06322714686393738, "p05": 0.043605200946331024, "p95": 0.08994095027446747 }, { "label": "completed_group_5", "mean": 0.08635804057121277, "median": 0.0823119580745697, "p05": 0.043849315494298935, "p95": 0.14317797124385834 }, { "label": "completed_group_6", "mean": 0.24226921796798706, "median": 0.23795896768569946, "p05": 0.11723512411117554, "p95": 0.3776393532752991 }, { "label": "completed_group_7", "mean": 0.14922890067100525, "median": 0.14130288362503052, "p05": 0.06516221165657043, "p95": 0.2617015242576599 }, { "label": "current_group_partial", "mean": 0.11574049293994904, "median": 0.08995743095874786, "p05": 0.03192901611328125, "p95": 0.2981606125831604 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1297550201416016, "group": 8, "latest_source_mass": 0.10419845581054688, "layer": 32, "max_source_mass": 0.17309343814849854, "mixer_index": 62, "normalized_entropy": 0.9692932812191579, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10085826367139816, "median": 0.10132397711277008, "p05": 0.025741146877408028, "p95": 0.20014458894729614 }, { "label": "completed_group_1", "mean": 0.08009681105613708, "median": 0.0792180746793747, "p05": 0.058846138417720795, "p95": 0.10435555130243301 }, { "label": "completed_group_2", "mean": 0.0982624888420105, "median": 0.09733013808727264, "p05": 0.0730227455496788, "p95": 0.12785884737968445 }, { "label": "completed_group_3", "mean": 0.11403533071279526, "median": 0.11354124546051025, "p05": 0.08656539022922516, "p95": 0.14382149279117584 }, { "label": "completed_group_4", "mean": 0.09301646053791046, "median": 0.09111075103282928, "p05": 0.06575828790664673, "p95": 0.12499162554740906 }, { "label": "completed_group_5", "mean": 0.12771157920360565, "median": 0.12228857725858688, "p05": 0.08901788294315338, "p95": 0.18423326313495636 }, { "label": "completed_group_6", "mean": 0.17309343814849854, "median": 0.17134559154510498, "p05": 0.12187358736991882, "p95": 0.2286318838596344 }, { "label": "completed_group_7", "mean": 0.10872717201709747, "median": 0.10637345165014267, "p05": 0.053436052054166794, "p95": 0.1774827241897583 }, { "label": "current_group_partial", "mean": 0.10419845581054688, "median": 0.09291182458400726, "p05": 0.04793589562177658, "p95": 0.21689072251319885 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1051812171936035, "group": 8, "latest_source_mass": 0.08249962329864502, "layer": 32, "max_source_mass": 0.21416787803173065, "mixer_index": 63, "normalized_entropy": 0.9581092615238248, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09203393757343292, "median": 0.08020778000354767, "p05": 0.047239016741514206, "p95": 0.20447812974452972 }, { "label": "completed_group_1", "mean": 0.08722756803035736, "median": 0.08618196845054626, "p05": 0.06572069227695465, "p95": 0.11272746324539185 }, { "label": "completed_group_2", "mean": 0.10295312851667404, "median": 0.1034228652715683, "p05": 0.06958446651697159, "p95": 0.13481324911117554 }, { "label": "completed_group_3", "mean": 0.11847956478595734, "median": 0.11627265810966492, "p05": 0.08390682935714722, "p95": 0.1599627137184143 }, { "label": "completed_group_4", "mean": 0.07401066273450851, "median": 0.07330383360385895, "p05": 0.05390108376741409, "p95": 0.09712604433298111 }, { "label": "completed_group_5", "mean": 0.11382973194122314, "median": 0.10948765277862549, "p05": 0.07062914222478867, "p95": 0.1708521842956543 }, { "label": "completed_group_6", "mean": 0.21416787803173065, "median": 0.21409451961517334, "p05": 0.1212707906961441, "p95": 0.3072466552257538 }, { "label": "completed_group_7", "mean": 0.11479790508747101, "median": 0.10708929598331451, "p05": 0.06293575465679169, "p95": 0.18719080090522766 }, { "label": "current_group_partial", "mean": 0.08249962329864502, "median": 0.07665085047483444, "p05": 0.049256522208452225, "p95": 0.13453297317028046 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.8321013450622559, "group": null, "latest_source_mass": 0.219556525349617, "layer": null, "max_source_mass": 0.30178403854370117, "mixer_index": null, "normalized_entropy": 0.8338252557157281, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09867185354232788, "median": 0.10158779472112656, "p05": 0.035266630351543427, "p95": 0.1513454020023346 }, { "label": "completed_group_1", "mean": 0.05898427963256836, "median": 0.05800648033618927, "p05": 0.04004587605595589, "p95": 0.0809728130698204 }, { "label": "completed_group_2", "mean": 0.057430200278759, "median": 0.05592039227485657, "p05": 0.03935804218053818, "p95": 0.08066077530384064 }, { "label": "completed_group_3", "mean": 0.05067210644483566, "median": 0.0480031743645668, "p05": 0.03069971315562725, "p95": 0.0778159350156784 }, { "label": "completed_group_4", "mean": 0.04944504797458649, "median": 0.04722342640161514, "p05": 0.030949361622333527, "p95": 0.07596699148416519 }, { "label": "completed_group_5", "mean": 0.047245003283023834, "median": 0.04242067039012909, "p05": 0.021957436576485634, "p95": 0.09008318185806274 }, { "label": "completed_group_6", "mean": 0.1162109300494194, "median": 0.09733854234218597, "p05": 0.03573312610387802, "p95": 0.2731606960296631 }, { "label": "completed_group_7", "mean": 0.30178403854370117, "median": 0.29523494839668274, "p05": 0.09429201483726501, "p95": 0.5453053712844849 }, { "label": "completed_group_8", "mean": 0.219556525349617, "median": 0.19983074069023132, "p05": 0.03722763806581497, "p95": 0.4634954333305359 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "942e0d186e83d3b52788e6a64c0684235f3d3e5e1f34cb38f76047fded397c74", "optimizer_state_before": "942e0d186e83d3b52788e6a64c0684235f3d3e5e1f34cb38f76047fded397c74", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "729d18a894ede5429f33682ebed5494f0e192bf108efbc8a123308e96fadab4d", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08130212873220444, 0.1432749330997467, 0.12958019971847534, 0.04409155994653702, 0.14100632071495056, 0.0481393076479435, 0.04208913818001747, 0.04236427694559097, 0.05848926678299904, 0.04504085332155228, 0.04397343844175339, 0.05665678158402443, 0.06478945910930634, 0.04678606986999512, 0.04817425087094307, 0.049439266324043274, 0.04376237466931343, 0.02850259095430374, 0.02922668866813183, 0.04021619260311127, 0.04409762844443321, 0.030135832726955414, 0.03803896903991699, 0.030100898817181587, 0.03295852243900299, 0.0306381918489933, 0.02206242084503174, 0.02074088715016842, 0.021799447014927864, 0.026608876883983612, 0.01902451552450657, 0.02519184537231922 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3376500305639638e-05, "first_to_last_ratio": 1.4098205594119886, "imbalance_abs_log_ratio": 0.3434624334630992, "last_quartile_mean": 9.48808713019389e-06, "mean": 1.3753251778325648e-05, "normalized": [ 0.872126231956151, 0.8994990546880632, 0.8826478210575978, 0.8463839625815878, 1.12968786363437, 1.1469805773400135, 1.040016753229815, 0.9635087058816061, 1.057144936913433, 0.941033957776458, 0.8736259813008125, 0.8166546176089677, 0.726185589501133, 0.6844792789299848, 0.6647323036859834, 0.638869348101534, 1.45211763876736, 1.3695497549648363, 1.2684543887383168, 1.1495085731929047, 2.0935002358588783, 1.9169888836924016, 1.6426233009262203, 1.404643849952257, 1.1986055749986184, 1.025360289254095, 0.8286550591551723, 0.7176699044021386, 0.5714448392683698, 0.43393140073490627, 0.38174836988061706, 0.36162095202539646 ], "peak_layer": 21, "peak_normalized": 2.0935002358588783, "population_cv": 0.3980646966565043, "rest_mean": 1.2094527173084552e-05, "spike_contrast": 1.8777389410615202, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.2710364646627568e-05, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 1.199457165057538e-05, 1.2371036973490845e-05, 1.2139277714595664e-05, 1.1640531738521531e-05, 1.55368816194823e-05, 1.577471266500652e-05, 1.4303612260846421e-05, 1.3251377822598442e-05, 1.4539180483552627e-05, 1.2942276953253895e-05, 1.2015198080916889e-05, 1.1231656571908388e-05, 9.987413250200916e-06, 9.413815860170871e-06, 9.142230737779755e-06, 8.78653099789517e-06, 1.9971339497715235e-05, 1.8835762602975592e-05, 1.7445372577640228e-05, 1.5809480828465894e-05, 2.8792435841751285e-05, 2.636483077367302e-05, 2.2591411834582686e-05, 1.9318420527270064e-05, 1.6484724255860783e-05, 1.4102038221608382e-05, 1.139670166594442e-05, 9.87029488896951e-06, 7.85922475188272e-06, 5.967967808828689e-06, 5.2502814469335135e-06, 4.973464001523098e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.305891409079777e-05, "first_to_last_ratio": 1.392774690210974, "imbalance_abs_log_ratio": 0.3312979374306361, "last_quartile_mean": 3.8095834497653414e-05, "mean": 5.492098136983259e-05, "normalized": [ 0.861703606856031, 0.8875085509159336, 0.8724772501581232, 0.8369715559103585, 1.1319676146266233, 1.1404157619380209, 1.0352725679446342, 0.9624472568066351, 1.064445563081042, 0.9432467955493706, 0.8769115703251862, 0.8183121449066616, 0.7308699981720386, 0.6902004800301615, 0.6672877834263703, 0.6409674903247492, 1.4550601983922828, 1.3755688672717281, 1.2726074279553434, 1.1425630054516895, 2.0822419693705934, 1.9086329329387715, 1.644199125843742, 1.4089356642756514, 1.2059871047522959, 1.0325249891655295, 0.8327095265990165, 0.7226098406524658, 0.5732866609317387, 0.43596455609961304, 0.38312094357080967, 0.3629811957567887 ], "peak_layer": 21, "peak_normalized": 2.0822419693705934, "population_cv": 0.3966570033750761, "rest_mean": 4.831012901612902e-05, "spike_contrast": 1.8757884925038641, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.061958407983184e-05, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 4.732560773845762e-05, 4.874284059042111e-05, 4.791730680153705e-05, 4.5967299229232594e-05, 6.216877227416262e-05, 6.263275281526148e-05, 5.6858185416786e-05, 5.2858547860523686e-05, 5.8460394939174876e-05, 5.1804039685521275e-05, 4.816084401682019e-05, 4.494250606512651e-05, 4.014009755337611e-05, 3.790648770518601e-05, 3.6648099921876565e-05, 3.52025635947939e-05, 7.991333404788747e-05, 7.554759213235229e-05, 6.989284884184599e-05, 6.275068153627217e-05, 0.00011435877240728587, 0.0001048239937517792, 9.030102955875918e-05, 7.738012936897576e-05, 6.623399531235918e-05, 5.670728569384664e-05, 4.57332243968267e-05, 3.968644159613177e-05, 3.148546602460556e-05, 2.3943601263454184e-05, 2.104137820424512e-05, 1.993528348975815e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04705317597836256, "first_to_last_ratio": 1.3927746759399198, "imbalance_abs_log_ratio": 0.33129792718414436, "last_quartile_mean": 0.03378376760520041, "mean": 0.04870447557186708, "normalized": [ 0.8617036111459598, 0.8875085515420177, 0.8724772009144255, 0.8369715628486869, 1.1319676319756757, 1.1404158445724537, 1.0352725718667957, 0.9624472384239273, 1.0644455689163006, 0.9432467761096968, 0.8769115717297861, 0.8183120975983121, 0.7308699765260276, 0.6902005050409423, 0.6672877917020854, 0.6409674776009149, 1.4550601346467584, 1.3755688270610786, 1.2726074407181456, 1.1425630055475084, 2.082241939293198, 1.9086328990500294, 1.6441992249175394, 1.4089356413043088, 1.2059871669112603, 1.032524982933128, 0.8327095186818301, 0.7226098470311895, 0.5732867026926071, 0.43596454525011596, 0.38312095710023625, 0.36298118834705717 ], "peak_layer": 21, "peak_normalized": 2.082241939293198, "population_cv": 0.3966570015078369, "rest_mean": 0.04284190545203509, "spike_contrast": 1.8757885152641456, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0803623542189598, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.04196882247924805, 0.04322563856840134, 0.0424935445189476, 0.04076426103711128, 0.05513188987970352, 0.05554335564374924, 0.05042240768671036, 0.0468754880130291, 0.05184326320886612, 0.0459403395652771, 0.04270951822400093, 0.039855461567640305, 0.035596638917922974, 0.033615853637456894, 0.032499901950359344, 0.03121798485517502, 0.07086794078350067, 0.06699635833501816, 0.0619816780090332, 0.05564793199300766, 0.1014145016670227, 0.092958964407444, 0.08007986098527908, 0.06862147152423859, 0.05873697251081467, 0.05028858780860901, 0.04055668041110039, 0.035194333642721176, 0.027921628206968307, 0.02123342454433441, 0.01865970529615879, 0.01767880842089653 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.0335509560709397e-06, "first_to_last_ratio": 0.800252825630956, "imbalance_abs_log_ratio": 0.2228275692031206, "last_quartile_mean": 2.541135614819723e-06, "mean": 2.913915704283454e-06, "normalized": [ 0.7803382065461789, 0.7320562648837056, 0.7098851296044338, 0.713059869815856, 0.6729150824257989, 0.6708688942390978, 0.6491951630171852, 0.6546869735527502, 0.7715235933317293, 0.7335599865229504, 0.7284058520495426, 0.7336793728603509, 0.6162135948862999, 0.6241820086654949, 0.6341697291979046, 0.6616511765677783, 1.3815401009435135, 1.3825758749583201, 1.3822762386605307, 1.332295187522962, 2.3406602841103434, 2.278960644638916, 2.0404236213013194, 1.7983249835948945, 1.517129115086387, 1.3281010596567688, 1.062368999113631, 0.926894533430433, 0.6999310401252395, 0.5293580836703028, 0.4667888356778559, 0.44598049934152495 ], "peak_layer": 21, "peak_normalized": 2.3406602841103434, "population_cv": 0.5165207800134821, "rest_mean": 2.3769459394989773e-06, "spike_contrast": 2.445807596004911, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.813552434119629e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.2738397547072964e-06, 2.1331502466637176e-06, 2.068545427391655e-06, 2.077796352750738e-06, 1.9608178263297305e-06, 1.9548554064385826e-06, 1.8916999806606327e-06, 1.907702653625165e-06, 2.248154714834527e-06, 2.137531964763184e-06, 2.1225132513791323e-06, 2.1378798464866122e-06, 1.7955944713321514e-06, 1.8188137573815766e-06, 1.8479171330909594e-06, 1.9279957541584736e-06, 4.0256913962366525e-06, 4.028709554404486e-06, 4.027836439490784e-06, 3.882195869664429e-06, 6.820486760261701e-06, 6.640699211857282e-06, 5.94562243350083e-06, 5.240167411102448e-06, 4.420786353875883e-06, 3.869974534609355e-06, 3.0956537102611037e-06, 2.700892537177424e-06, 2.039540049736388e-06, 1.5425048331962898e-06, 1.360183318865893e-06, 1.2995495808354462e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.0862232708605006e-05, "first_to_last_ratio": 1.6228918616520855, "imbalance_abs_log_ratio": 0.48420965763249596, "last_quartile_mean": 2.5178654027513403e-05, "mean": 3.966477586914152e-05, "normalized": [ 0.9416433745088736, 0.9689750084042642, 0.950763641261749, 0.911671542713804, 1.193012300282416, 1.190876368625092, 1.0841056502696969, 1.00046752668501, 1.1216839411912192, 0.9926384676731087, 0.9187729101347912, 0.8506497325833076, 0.7805624114195948, 0.7377957179764669, 0.7154649230403652, 0.6867945255570269, 1.3962697506908583, 1.3266925657293973, 1.2247822658863927, 1.1061834866215106, 2.1088522482676924, 1.872182986128609, 1.5438875021669929, 1.296981213404733, 1.145045831270383, 0.9535938791130757, 0.7733123689206051, 0.6739013879397645, 0.49031224706376303, 0.37696827424181495, 0.33882165196875497, 0.32633429825886623 ], "peak_layer": 21, "peak_normalized": 2.1088522482676924, "population_cv": 0.38950134534158565, "rest_mean": 3.530613149551856e-05, "spike_contrast": 1.7900985695566152, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.320145548670553e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 3.735007339855656e-05, 3.843417653115466e-05, 3.7711826735176146e-05, 3.6161247408017516e-05, 4.732056549983099e-05, 4.723584424937144e-05, 4.300080763641745e-05, 3.968332021031529e-05, 4.449134212336503e-05, 3.937278233934194e-05, 3.64429215551354e-05, 3.3740830986062065e-05, 3.096083310083486e-05, 2.926450179074891e-05, 2.8378755814628676e-05, 2.724155092437286e-05, 5.538272671401501e-05, 5.262296326691285e-05, 4.858071406488307e-05, 4.387652006698772e-05, 8.364715176867321e-05, 7.425971853081137e-05, 6.123795174062252e-05, 5.1444469136185944e-05, 4.541798625723459e-05, 3.782408748520538e-05, 3.067326179007068e-05, 2.6730147510534152e-05, 1.9448125385679305e-05, 1.4952362107578665e-05, 1.343928488495294e-05, 1.2943976798851509e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.1457733257411746e-05, "first_to_last_ratio": 1.8977474170968827, "imbalance_abs_log_ratio": 0.6406676128793697, "last_quartile_mean": 1.657635414176184e-05, "mean": 2.8327688568197118e-05, "normalized": [ 1.0378057368348226, 1.06983861568018, 1.0491977904887932, 1.0092011590321537, 1.2852683271687153, 1.2533015871147788, 1.1416273294114696, 1.0377128856771576, 1.1745255348133223, 1.0416840385088497, 0.9634808894084209, 0.8849637415797892, 0.8443903541769905, 0.7965684762965399, 0.7722003128278548, 0.7374273616451005, 1.3424651539896326, 1.2854458102944317, 1.1706594334988774, 1.0674068913781267, 2.103772472082861, 1.7707976898599065, 1.3606069602640058, 1.1183364576939212, 1.065869260796216, 0.8913709209455979, 0.7395545264445227, 0.6679526066396835, 0.41071849146647305, 0.32387129128319875, 0.29724047481130667, 0.2847374178863 ], "peak_layer": 21, "peak_normalized": 2.103772472082861, "population_cv": 0.379228075638394, "rest_mean": 2.5789335840845188e-05, "spike_contrast": 1.6299292682567954, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2034793295897543e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 2.9398637707345188e-05, 3.0306055123219267e-05, 2.9721348255407065e-05, 2.8588336135726422e-05, 3.640868089860305e-05, 3.5503137041814625e-05, 3.233966344851069e-05, 2.939600744866766e-05, 3.3271593565586954e-05, 2.950850102934055e-05, 2.7293186576571316e-05, 2.506897726561874e-05, 2.391962698311545e-05, 2.256494371977169e-05, 2.1874649974051863e-05, 2.0889612642349675e-05, 3.80289347958751e-05, 3.641370858531445e-05, 3.3162075851578265e-05, 3.0237169994506985e-05, 5.9595011407509446e-05, 5.0162605475634336e-05, 3.8542850234080106e-05, 3.167988688801415e-05, 3.0193612474249676e-05, 2.525047784729395e-05, 2.094987030432094e-05, 1.892155341920443e-05, 1.1634705515461974e-05, 9.174525075650308e-06, 8.420135600317735e-06, 8.065952897595707e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010518337694520596, "first_to_last_ratio": 1.298853979021645, "imbalance_abs_log_ratio": 0.2614823210709866, "last_quartile_mean": 8.098167973003001e-05, "mean": 0.0001129772994090672, "normalized": [ 0.810962534989689, 0.8287901520895661, 0.8193344011879904, 0.7967594592504931, 1.090444814007616, 1.1188741501855428, 1.0312115132038047, 0.9517299838647473, 1.0314031089846865, 0.9115982349983334, 0.8441619298911692, 0.7979269376759806, 0.6929773379071392, 0.654575232028, 0.6380663091161348, 0.6205945771541241, 1.4091870327874871, 1.3398512567689524, 1.2663035936681761, 1.1428583354431259, 2.1533778907233563, 2.0039677010770927, 1.7844997896295434, 1.5261754809958494, 1.2411778195662533, 1.0739169569244635, 0.8779998056570737, 0.7656704516242112, 0.5726664578534288, 0.445094451914986, 0.3923455251595323, 0.36549677367145117 ], "peak_layer": 21, "peak_normalized": 2.1533778907233563, "population_cv": 0.4230707862688655, "rest_mean": 9.745673459191393e-05, "spike_contrast": 2.019238077755404, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019678834942169488, "top_five_layers": [ 21, 22, 23, 24, 17 ], "top_five_spike_overlap": 4 }, "values": [ 9.162035712506622e-05, 9.363447315990925e-05, 9.256618795916438e-05, 9.001573198474944e-05, 0.000123195510241203, 0.00012640737986657768, 0.00011650349188130349, 0.00010752388334367424, 0.00011652513785520568, 0.00010298990673618391, 9.537113510305062e-05, 9.014763054437935e-05, 7.82907081884332e-05, 7.3952141974587e-05, 7.208700844785199e-05, 7.011309935478494e-05, 0.00015920614532660693, 0.0001513727765996009, 0.0001430635602446273, 0.0001291170483455062, 0.00024328281870111823, 0.0002264028589706868, 0.00020160796702839434, 0.00017242318426724523, 0.00014022491814102978, 0.00012132823758292943, 9.919404692482203e-05, 8.650337986182421e-05, 6.469830987043679e-05, 5.0285569159314036e-05, 4.432613786775619e-05, 4.129283843212761e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "0c4b69b5a62d474637d9a6156b8452d836a142d489dd60b2de0cdb09db585af9", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13576549291610718, 0.041234951466321945, 0.023600688204169273, 0.0554339736700058, 0.01666521653532982, 0.008341255597770214, 0.01377889234572649, 0.03695881366729736, 0.033706482499837875, 0.0386466309428215, 0.04499480128288269, 0.057638294994831085, 0.04709118604660034, 0.044345736503601074, 0.04982399195432663, 0.060589101165533066, 0.040811967104673386, 0.04198655113577843, 0.05064959079027176, 0.07217346876859665, 0.039372652769088745, 0.04702961444854736, 0.054020222276449203, 0.06186771020293236, 0.054687224328517914, 0.05866995081305504, 0.06297128647565842, 0.07938925921916962, 0.07573848962783813, 0.1105475053191185, 0.12047550827264786, 0.16982334852218628 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.32639113417099e-05, "first_to_last_ratio": 1.6142215524258654, "imbalance_abs_log_ratio": 0.4788528295882149, "last_quartile_mean": 8.216908838676318e-06, "mean": 1.2790947593543933e-05, "normalized": [ 0.9316524047936077, 0.9631591197877182, 0.9539238462095513, 0.9083131861987929, 1.2160660233571214, 1.219529953106465, 1.0980857820278223, 1.0050812307438088, 1.0448625967092706, 0.9611285157606685, 0.8964095006895693, 0.8477627470675287, 0.7494675898854604, 0.706090181898008, 0.6924087401774147, 0.671952384866888, 1.468062717452813, 1.366424579378114, 1.2391161308106104, 1.1215385546051009, 2.1279101909054225, 1.8304940369610547, 1.5253256372483166, 1.316031782178504, 1.115126772003547, 0.8926197698458935, 0.7767158527757819, 0.6977024035715537, 0.4688772635179848, 0.4106255281575897, 0.3886908734905123, 0.38884410381750617 ], "peak_layer": 21, "peak_normalized": 2.1279101909054225, "population_cv": 0.387928054896289, "rest_mean": 1.1410051852349338e-05, "spike_contrast": 1.7745567555703712, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0247784595994746e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.1916717085114215e-05, 1.2319717825448606e-05, 1.2201589925098233e-05, 1.1618186363193672e-05, 1.5554636775050312e-05, 1.5598943718941882e-05, 1.4045557691133581e-05, 1.2855941349698696e-05, 1.3364782716962509e-05, 1.2293744475755375e-05, 1.1465926945675164e-05, 1.0843688869499601e-05, 9.5864006652846e-06, 9.031562512973323e-06, 8.856563908921089e-06, 8.594907740189228e-06, 1.8777913282974623e-05, 1.7477865185355768e-05, 1.5849469491513446e-05, 1.4345540876092855e-05, 2.7217987735639326e-05, 2.3413753297063522e-05, 1.9510360289132223e-05, 1.683329355728347e-05, 1.4263528100855183e-05, 1.1417452697060071e-05, 9.934931767929811e-06, 8.924274879973382e-06, 5.997384505462833e-06, 5.252289611235028e-06, 4.971724592905957e-06, 4.973684553988278e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.259526642475976e-05, "first_to_last_ratio": 1.5935840922413673, "imbalance_abs_log_ratio": 0.465985625520115, "last_quartile_mean": 3.3004387205437524e-05, "mean": 5.1062648935840116e-05, "normalized": [ 0.9216268503960833, 0.950589815056072, 0.9422144196635877, 0.8985231786108131, 1.2178691387217164, 1.2132293533449192, 1.0941896193182434, 1.0018732508280441, 1.0531925942812075, 0.964189208098705, 0.8991854758416759, 0.850007127879547, 0.7546609062980287, 0.7123005293021083, 0.6948749020960197, 0.6737990168624725, 1.4721384112868208, 1.371930474910031, 1.2436471490563514, 1.1111982472028779, 2.113050053165766, 1.8242395861657554, 1.5311286043798866, 1.3195351834383122, 1.1232494064757303, 0.8974402485431373, 0.7821413966916887, 0.7035327854962891, 0.4712604586854498, 0.41227083555557753, 0.3905192233334204, 0.390392549013662 ], "peak_layer": 21, "peak_normalized": 2.113050053165766, "population_cv": 0.38599887162413726, "rest_mean": 4.5556955296046067e-05, "spike_contrast": 1.773459048474649, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.079339459072798e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.706070831161924e-05, 4.853963400819339e-05, 4.811196413356811e-05, 4.5880973630119115e-05, 6.218762428034097e-05, 6.195070454850793e-05, 5.5872220400488004e-05, 5.11583020852413e-05, 5.377880370360799e-05, 4.923405504086986e-05, 4.591479228110984e-05, 4.3403615563875064e-05, 3.8534984923899174e-05, 3.6371951864566654e-05, 3.548215318005532e-05, 3.440596265136264e-05, 7.517128688050434e-05, 7.005440420471132e-05, 6.350391777232289e-05, 5.674072599504143e-05, 0.0001078979330486618, 9.315050556324422e-05, 7.818348240107298e-05, 6.737896183039993e-05, 5.735609011026099e-05, 4.582567635225132e-05, 3.993821155745536e-05, 3.592424764065072e-05, 2.406380735919811e-05, 2.1051640942459926e-05, 1.9940946003771387e-05, 1.9934477677452378e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.046642007771879435, "first_to_last_ratio": 1.5935840941315902, "imbalance_abs_log_ratio": 0.46598562670626065, "last_quartile_mean": 0.029268620302900672, "mean": 0.04528286767890677, "normalized": [ 0.9216268273678067, 0.9505898655982172, 0.9422144164682038, 0.8985231812169583, 1.2178690166672288, 1.2132293989104181, 1.0941896425839293, 1.0018732026333346, 1.053192572659169, 0.9641892757112167, 0.8991854313888171, 0.8500071449620881, 0.7546608899084634, 0.7123004895677968, 0.694874918803495, 0.6737989921850662, 1.4721383659439504, 1.3719304542249628, 1.2436471663071618, 1.1111982836748173, 2.1130500968616075, 1.824239588496973, 1.531128649637463, 1.3195352773050415, 1.1232494269264028, 0.8974402993520915, 0.7821413926920331, 0.7035327086278959, 0.4712604718453035, 0.41227080515505665, 0.3905192188271276, 0.39039252748990244 ], "peak_layer": 21, "peak_normalized": 2.1130500968616075, "population_cv": 0.3859988823671551, "rest_mean": 0.04040036315995234, "spike_contrast": 1.7734591097013597, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07164839208126068, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.04173390567302704, 0.04304543510079384, 0.04266617074608803, 0.04068770632147789, 0.05514860153198242, 0.054938506335020065, 0.04954804480075836, 0.04536769166588783, 0.04769157990813255, 0.04366125538945198, 0.04071769490838051, 0.03849076107144356, 0.034173209220170975, 0.032255008816719055, 0.03146592900156975, 0.03051155060529709, 0.06666264683008194, 0.06212494522333145, 0.05631591007113457, 0.050318244844675064, 0.0956849679350853, 0.08260679990053177, 0.06933389604091644, 0.059752341359853745, 0.05086395516991615, 0.040638670325279236, 0.03541760519146919, 0.03185797855257988, 0.02134002558887005, 0.018668804317712784, 0.017683830112218857, 0.017678093165159225 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.0265283637854736e-06, "first_to_last_ratio": 0.9156476671615019, "imbalance_abs_log_ratio": 0.08812363113882582, "last_quartile_mean": 2.2132185080181443e-06, "mean": 2.6874861127623717e-06, "normalized": [ 0.8352375485728037, 0.7929371121311408, 0.7730150142183774, 0.7741669057859127, 0.7344628997751053, 0.7231354442993345, 0.6957352730984985, 0.7037974988161008, 0.790273167233088, 0.7708879700832757, 0.7554230960744762, 0.7732858335285265, 0.6462830832935508, 0.6542078265434891, 0.6700938622286156, 0.7131401727840788, 1.4047798643220162, 1.3792830850081157, 1.3465103950870556, 1.2824167454988713, 2.3822796171807563, 2.1737357192581888, 1.9010524006612797, 1.735640094726425, 1.463796392135114, 1.1467668979777017, 1.0088922880136448, 0.9362270084384399, 0.5805339755855011, 0.495871510556662, 0.4753741013945969, 0.4807571956892568 ], "peak_layer": 21, "peak_normalized": 2.3822796171807563, "population_cv": 0.48438790023185435, "rest_mean": 2.2239938743950916e-06, "spike_contrast": 2.3337942877011817, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.190344199945684e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.244689312647097e-06, 2.1310074771463405e-06, 2.0774671156686964e-06, 2.080562808259856e-06, 1.973858843484777e-06, 1.943416464200709e-06, 1.8697788846111507e-06, 1.8914460042651626e-06, 2.1238481622276595e-06, 2.071750714094378e-06, 2.0301890799601097e-06, 2.07819493880379e-06, 1.7368768112646649e-06, 1.7581744486960815e-06, 1.8008679489867063e-06, 1.91655431081017e-06, 3.775326376853627e-06, 3.706804136527353e-06, 3.6187279874866363e-06, 3.4464771943021333e-06, 6.4023433878901415e-06, 5.841884558321908e-06, 5.109051926410757e-06, 4.664508651330834e-06, 3.933932475774782e-06, 3.081920112890657e-06, 2.7113840133097256e-06, 2.516097083571367e-06, 1.5601769973727642e-06, 1.332647798335529e-06, 1.2775612958648708e-06, 1.2920282870254596e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.059575303472229e-05, "first_to_last_ratio": 1.8428664869924756, "imbalance_abs_log_ratio": 0.6113222327604917, "last_quartile_mean": 2.20285915020213e-05, "mean": 3.723078083339715e-05, "normalized": [ 1.000747308098028, 1.0297510550328743, 1.0169292808330181, 0.9702628967336459, 1.2715949638568862, 1.257179274381204, 1.1379428423791025, 1.0386440313235143, 1.1094267873975205, 1.0094867752605219, 0.9345792064389409, 0.8772088096510522, 0.8023959259632547, 0.7583457927560233, 0.7417727186299412, 0.7178167934970746, 1.4137765645172868, 1.3271740625307213, 1.203509903524359, 1.0853458857052019, 2.1304053478424945, 1.7810705561765154, 1.4345481768124502, 1.2166701599774927, 1.051280441541134, 0.8337316241032007, 0.7246574768415295, 0.6567125387812885, 0.4057821793994631, 0.362051816161588, 0.34956720507227845, 0.3496315987803933 ], "peak_layer": 21, "peak_normalized": 2.1304053478424945, "population_cv": 0.3839892025017025, "rest_mean": 3.3626324592559184e-05, "spike_contrast": 1.6860256129945155, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.669484453392215e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 3.725860369740985e-05, 3.833843584288843e-05, 3.7861071177758276e-05, 3.6123645259067416e-05, 4.73424734082073e-05, 4.6805766032775864e-05, 4.236650056554936e-05, 3.8669528294121847e-05, 4.130482557229698e-05, 3.758398088393733e-05, 3.4795113606378436e-05, 3.265916893724352e-05, 2.98738268611487e-05, 2.8233806006028317e-05, 2.761677751550451e-05, 2.6724879717221484e-05, 5.263600542093627e-05, 4.941172664985061e-05, 4.4807613448938355e-05, 4.040827479911968e-05, 7.931665459182113e-05, 6.63106475258246e-05, 5.340934876585379e-05, 4.5297580072656274e-05, 3.913999171345495e-05, 3.104047937085852e-05, 2.6979563699569553e-05, 2.444992060190998e-05, 1.5107587387319654e-05, 1.3479471817845479e-05, 1.3014659998589195e-05, 1.3017057426623069e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.131921857857378e-05, "first_to_last_ratio": 2.116701522335747, "imbalance_abs_log_ratio": 0.7498589913461419, "last_quartile_mean": 1.4796237564951298e-05, "mean": 2.6929862087854417e-05, "normalized": [ 1.0947226696497885, 1.1271282027493932, 1.106919822700648, 1.0632330527264946, 1.3526853947996393, 1.3079534266756163, 1.1814225789176866, 1.069872161641373, 1.157814771927838, 1.0519056823159265, 0.9685157022801635, 0.9017704048401847, 0.8577058532160233, 0.8140538691438461, 0.7944030091628936, 0.7643614989434289, 1.351455932687456, 1.2802515554321876, 1.1546255463982888, 1.0491990014149588, 2.127009711159991, 1.6881429365545844, 1.2776017477923152, 1.0617569146647619, 0.9776304191432695, 0.792423657567716, 0.7111660870958268, 0.6578041081913825, 0.34259196138670317, 0.31373499556133577, 0.3002789981315749, 0.299858325126703 ], "peak_layer": 21, "peak_normalized": 2.127009711159991, "population_cv": 0.3819549344885706, "rest_mean": 2.4803259024068853e-05, "spike_contrast": 1.548728680977784, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.841351863229647e-05, "top_five_layers": [ 21, 22, 5, 17, 6 ], "top_five_spike_overlap": 2 }, "values": [ 2.9480730518116616e-05, 3.0353407055372372e-05, 2.9809198167640716e-05, 2.863271947717294e-05, 3.642763113020919e-05, 3.522300539771095e-05, 3.18155471177306e-05, 2.8811509764636867e-05, 3.117979213129729e-05, 2.83276749541983e-05, 2.608199429232627e-05, 2.428455263725482e-05, 2.3097900339053012e-05, 2.1922358428128064e-05, 2.1393163478933275e-05, 2.058414975181222e-05, 3.639452188508585e-05, 3.4476997825549915e-05, 3.109390672761947e-05, 2.825478441081941e-05, 5.728007818106562e-05, 4.546145646600053e-05, 3.440563887124881e-05, 2.859296728274785e-05, 2.6327452360419556e-05, 2.1339859813451767e-05, 1.915160464704968e-05, 1.7714573914417997e-05, 9.225954272551462e-06, 8.44884016260039e-06, 8.086472007562406e-06, 8.075143341557123e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00010410473532829201, "first_to_last_ratio": 1.4802470022359204, "imbalance_abs_log_ratio": 0.39220896725351806, "last_quartile_mean": 7.032929988781689e-05, "mean": 0.00010518467865949788, "normalized": [ 0.8620082316979699, 0.8793601137677285, 0.8838341652030255, 0.853808582522237, 1.1744815620635005, 1.1878682305333963, 1.0874202108294881, 0.9890819727977851, 1.0219046417637305, 0.9359499228966669, 0.8632032673756425, 0.8297183344841335, 0.7158097001920478, 0.6803843215474668, 0.6659599171875242, 0.6502865912508059, 1.4528893199520632, 1.3424784269798153, 1.242892858354237, 1.1348824127127712, 2.1809149327188573, 1.9248156603663433, 1.6620583320953677, 1.4289735266476418, 1.1532148244613336, 0.9405995358363591, 0.8280572099898275, 0.7404736013254825, 0.4795450387252912, 0.4212381094212713, 0.392435197099403, 0.3934512472007861 ], "peak_layer": 21, "peak_normalized": 2.1809149327188573, "population_cv": 0.40887132992350467, "rest_mean": 9.213407557345581e-05, "spike_contrast": 1.906546890830614, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000175657935324125, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 9.067005885299295e-05, 9.249521099263802e-05, 9.296581265516579e-05, 8.980758138932288e-05, 0.0001235374656971544, 0.00012494553811848164, 0.00011437994544394314, 0.00010403626947663724, 0.00010748871136456728, 9.844759188126773e-05, 9.079575829673558e-05, 8.727365639060736e-05, 7.529221329605207e-05, 7.156600622693077e-05, 7.004877988947555e-05, 6.840018613729626e-05, 0.00015282169624697417, 0.00014120816194918007, 0.0001307332859141752, 0.0001193722418975085, 0.00022939883638173342, 0.00020246111671440303, 0.00017482307157479227, 0.00015030612121336162, 0.00012130053073633462, 9.89366599242203e-05, 8.709893154446036e-05, 7.788647781126201e-05, 5.044079080107622e-05, 4.430779517861083e-05, 4.1278170101577416e-05, 4.138504300499335e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4cec366fa91114ffbf2b3855220297ceecccb362041740eb1c3b94ecdc658134", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08130212873220444, 0.2561342120170593, 0.3685380816459656, 0.38441672921180725, 0.14100632071495056, 0.16686734557151794, 0.18023288249969482, 0.18965557217597961, 0.05848926678299904, 0.09072534739971161, 0.12624414265155792, 0.17554737627506256, 0.06478945910930634, 0.11512516438961029, 0.15369470417499542, 0.19735252857208252, 0.04376237466931343, 0.07002050429582596, 0.09636416286230087, 0.12901709973812103, 0.04409762844443321, 0.07137759029865265, 0.10297419875860214, 0.1298016905784607, 0.03295852243900299, 0.07529281079769135, 0.11068660020828247, 0.1495535671710968, 0.021799447014927864, 0.08539005368947983, 0.17447474598884583, 0.27695441246032715 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3376565448197653e-05, "first_to_last_ratio": 1.4098290045963058, "imbalance_abs_log_ratio": 0.343468423699918, "last_quartile_mean": 9.488076500474563e-06, "mean": 1.3753222575019208e-05, "normalized": [ 0.8721340354721765, 0.8994799354622096, 0.8826639131112579, 0.8463853629982472, 1.1296999173073485, 1.1469756063027932, 1.0400388665804514, 0.9635277470755461, 1.0571451316162943, 0.9410534141549793, 0.8736341847787974, 0.8166490774236239, 0.7261875943754862, 0.6844771613422266, 0.6647247876726992, 0.6388650175203425, 1.452098767146314, 1.3695701212448788, 1.2684285141735008, 1.1495085011117947, 2.0934894713524743, 1.9169745701660839, 1.6426207049178487, 1.4046256710812022, 1.198612087866968, 1.0253598212999628, 0.8286524541524178, 0.7176736105632963, 0.5714418203699311, 0.43393410763421036, 0.3817475933674649, 0.36162043035717156 ], "peak_layer": 21, "peak_normalized": 2.0934894713524743, "population_cv": 0.39806173416327506, "rest_mean": 1.2094521530848902e-05, "spike_contrast": 1.8777268828380727, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.2710208213538863e-05, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 1.199465350509854e-05, 1.2370747754175682e-05, 1.2139473255956545e-05, 1.164052628155332e-05, 1.553701440570876e-05, 1.577461080159992e-05, 1.4303886018751655e-05, 1.3251611562736798e-05, 1.4539152289216872e-05, 1.2942517059855163e-05, 1.201528539240826e-05, 1.1231556527491193e-05, 9.987419616663828e-06, 9.413766747456975e-06, 9.142107955995016e-06, 8.786452781350818e-06, 1.9971037545474246e-05, 1.883600270957686e-05, 1.7444979675929062e-05, 1.5809446267667226e-05, 2.8792226657969877e-05, 2.6364577934145927e-05, 2.2591328161070123e-05, 1.9318129488965496e-05, 1.648477882554289e-05, 1.410200184182031e-05, 1.1396641639294103e-05, 9.87032490229467e-06, 7.859166544221807e-06, 5.967992365185637e-06, 5.250259619060671e-06, 4.973446266376413e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.305920649334439e-05, "first_to_last_ratio": 1.392778608294149, "imbalance_abs_log_ratio": 0.3313007505760008, "last_quartile_mean": 3.8095937270554714e-05, "mean": 5.492100513038167e-05, "normalized": [ 0.8617029028549882, 0.8875029339754659, 0.8724915117830877, 0.8369809973610108, 1.131954274301571, 1.1404132813521581, 1.0352898061888103, 0.9624677060873116, 1.0644389422291614, 0.9432618214397073, 0.8769178812067566, 0.8183058955006484, 0.7308681584499725, 0.6902021023938238, 0.6672861699325884, 0.6409658882179515, 1.4550351924910432, 1.3755767509041323, 1.2725799838609873, 1.142576421586798, 2.0822225212691614, 1.908623231016431, 1.6442116625532532, 1.4089265759792677, 1.2059942668685917, 1.0325358032982117, 0.8327031384838283, 0.7226180730156292, 0.5732837633016289, 0.4359643343675707, 0.38311796261183234, 0.36298004511661974 ], "peak_layer": 21, "peak_normalized": 2.0822225212691614, "population_cv": 0.3966538790797015, "rest_mean": 4.8310187628009805e-05, "spike_contrast": 1.87578281295371, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.061941964318975e-05, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 4.7325589548563585e-05, 4.874255319009535e-05, 4.791811079485342e-05, 4.5967837650096044e-05, 6.216806650627404e-05, 6.263264367589727e-05, 5.6859156757127494e-05, 5.285969382384792e-05, 5.8460056607145816e-05, 5.180488733458333e-05, 4.816121145267971e-05, 4.494218228501268e-05, 4.0140013879863545e-05, 3.7906593206571415e-05, 3.664802716230042e-05, 3.520249083521776e-05, 7.991199527168646e-05, 7.55480577936396e-05, 6.989137182245031e-05, 6.275144551182166e-05, 0.00011435775377321988, 0.00010482350626261905, 9.03017571545206e-05, 7.737966370768845e-05, 6.62344173179008e-05, 5.670790415024385e-05, 4.5732893340755254e-05, 3.968691089539789e-05, 3.1485320505453274e-05, 2.394359944446478e-05, 2.104122359014582e-05, 1.9935228920076042e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04705343535169959, "first_to_last_ratio": 1.3927786002877673, "imbalance_abs_log_ratio": 0.33130074482750527, "last_quartile_mean": 0.03378385864198208, "mean": 0.048704497050493956, "normalized": [ 0.8617028486976422, 0.8875029589946798, 0.8724915017730162, 0.836980984158547, 1.131954282861735, 1.1404132764850687, 1.0352898604369885, 0.9624676951017845, 1.064438980488691, 0.9432618106350297, 0.8769178394343718, 0.8183058471780916, 0.7308681244611269, 0.6902021128534255, 0.6672861971396676, 0.6409658946457253, 1.4550351699087516, 1.3755767870484994, 1.272579955863019, 1.1425763459349243, 2.0822225110266683, 1.908623337758237, 1.6442116556955237, 1.4089266063288464, 1.2059942838327453, 1.032535771268638, 0.8327031854247592, 0.7226180949727652, 0.5732837728076834, 0.4359643147464535, 0.3831179581029778, 0.36298003393391687 ], "peak_layer": 21, "peak_normalized": 2.0822225110266683, "population_cv": 0.39665388748012176, "rest_mean": 0.042841957675086126, "spike_contrast": 1.8757828549097615, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08036220967769622, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.041968803852796555, 0.04322538524866104, 0.042494259774684906, 0.040764737874269485, 0.05513126403093338, 0.05554325506091118, 0.0504232719540596, 0.04687650501728058, 0.05184296518564224, 0.04594109207391739, 0.0427098423242569, 0.03985517472028732, 0.035596564412117004, 0.033615946769714355, 0.03249983862042427, 0.031217921525239944, 0.07086675614118576, 0.06699677556753159, 0.061980366706848145, 0.05564860627055168, 0.10141360014677048, 0.09295853972434998, 0.08008050173521042, 0.06862106174230576, 0.05873734503984451, 0.05028913542628288, 0.04055638983845711, 0.035194750875234604, 0.02792149782180786, 0.021233422681689262, 0.018659567460417747, 0.01767875999212265 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.0335669290716396e-06, "first_to_last_ratio": 0.8002578806956262, "imbalance_abs_log_ratio": 0.22282125238855607, "last_quartile_mean": 2.5411395228047695e-06, "mean": 2.9139069361860948e-06, "normalized": [ 0.7803650562078773, 0.7320955321763626, 0.70991137711656, 0.7130517934518413, 0.6728847245961602, 0.6708950243493448, 0.649173395200872, 0.6546893336924704, 0.7715659445434139, 0.7335472900136112, 0.7283727740652475, 0.7337376064535494, 0.6162116646247685, 0.6242056183615969, 0.6341550559619227, 0.6616468470383687, 1.3816834645183462, 1.3825650533409872, 1.382166161319272, 1.3322845267305112, 2.3405652633576954, 2.2789024246956493, 2.040432570140748, 1.7983076099361612, 1.5171012195101121, 1.3281261242224407, 1.0623618177778928, 0.9268841353462449, 0.6999640463402925, 0.5293512882553055, 0.46681517102107106, 0.44598008563330255 ], "peak_layer": 21, "peak_normalized": 2.3405652633576954, "population_cv": 0.5165053196020641, "rest_mean": 2.3769592450696144e-06, "spike_contrast": 2.4457400690708226, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.813424468215089e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.2739111500413856e-06, 2.133258249159553e-06, 2.0686156858573668e-06, 2.077766566799255e-06, 1.9607234662544215e-06, 1.9549256649042945e-06, 1.891630859063298e-06, 1.9077037904935423e-06, 2.2482713575300295e-06, 2.1374885363911744e-06, 2.122410478477832e-06, 2.1380431007855805e-06, 1.7955834437088924e-06, 1.8188770809501875e-06, 1.8478688161849277e-06, 1.9279773368907627e-06, 4.026097030873643e-06, 4.028665898658801e-06, 4.027503564429935e-06, 3.8821531234134454e-06, 6.820189355494222e-06, 6.640509582211962e-06, 5.945630618953146e-06, 5.240101017989218e-06, 4.4206917664268985e-06, 3.870035925501725e-06, 3.0956234695622697e-06, 2.700854111026274e-06, 2.0396300897118635e-06, 1.5424803905261797e-06, 1.360255964755197e-06, 1.2995444649277488e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.08623318435275e-05, "first_to_last_ratio": 1.6228911311744263, "imbalance_abs_log_ratio": 0.48420920752374813, "last_quartile_mean": 2.517872644602903e-05, "mean": 3.966481972383917e-05, "normalized": [ 0.941643984322134, 0.9689728364581338, 0.9507669925320091, 0.9116714519207078, 1.193004560988191, 1.1908787206748392, 1.084110505037148, 1.0004772432625977, 1.1216820589953018, 0.9926455330841178, 0.9187755630308546, 0.8506477831794215, 0.7805623738669399, 0.7377941685016672, 0.7154642237174069, 0.6867938120739691, 1.3962577510757062, 1.3266966019768376, 1.2247798111115489, 1.1061916188277885, 2.108839277362418, 1.87217926525862, 1.5438876295558621, 1.296977303034086, 1.145052086148766, 0.9535982361523758, 0.7733129814099837, 0.6739034402524328, 0.4903104209064994, 0.3769676510880434, 0.33882058947149796, 0.3263335247220941 ], "peak_layer": 21, "peak_normalized": 2.108839277362418, "population_cv": 0.38949977839098365, "rest_mean": 3.530619142111391e-05, "spike_contrast": 1.790094315320562, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.32014125585556e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 3.735013888217509e-05, 3.843413287540898e-05, 3.7712001358158886e-05, 3.616128378780559e-05, 4.7320310841314495e-05, 4.723598976852372e-05, 4.3001047743018717e-05, 3.9683749491814524e-05, 4.449131665751338e-05, 3.937310611945577e-05, 3.644306707428768e-05, 3.374079096829519e-05, 3.0960865842644125e-05, 2.9264472686918452e-05, 2.8378759452607483e-05, 2.7241552743362263e-05, 5.5382311984431e-05, 5.262318154564127e-05, 4.858067040913738e-05, 4.3876891140826046e-05, 8.364672976313159e-05, 7.425965304719284e-05, 6.123802450019866e-05, 5.144437091075815e-05, 4.541828457149677e-05, 3.7824302125955e-05, 3.0673319997731596e-05, 2.673025846888777e-05, 1.9448074453976005e-05, 1.4952353922126349e-05, 1.3439257600111887e-05, 1.2943960427946877e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.14603882998199e-05, "first_to_last_ratio": 1.8980819473898056, "imbalance_abs_log_ratio": 0.6408438749089113, "last_quartile_mean": 1.6574831420257397e-05, "mean": 2.8326866441830134e-05, "normalized": [ 1.0377835864880585, 1.0701104047897898, 1.0491032802041116, 1.0093792976317837, 1.285175659609536, 1.2536418234821836, 1.1421213928087102, 1.0376456542092856, 1.1747807768794196, 1.0416012540102209, 0.9636221263630744, 0.8849957187485022, 0.8443134022244022, 0.796513189344223, 0.7723615555596486, 0.7373587354640148, 1.3422069324537358, 1.2852755770330793, 1.1704786768102076, 1.0672891502889117, 2.103806816412106, 1.770591712609653, 1.360589169814368, 1.1182332944394449, 1.0655109283784003, 0.891301818145435, 0.7396229310643554, 0.6680012742401278, 0.4107558405430897, 0.3238447630597274, 0.29727709900388205, 0.2847061578865119 ], "peak_layer": 21, "peak_normalized": 2.103806816412106, "population_cv": 0.37921215100884365, "rest_mean": 2.5789270289856682e-05, "spike_contrast": 1.6297431137094942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.202988566248678e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 2.93971570499707e-05, 3.031287451449316e-05, 2.9717808502027765e-05, 2.859255255316384e-05, 3.6404999264050275e-05, 3.5511744499672204e-05, 3.2352720154449344e-05, 2.9393249860731885e-05, 3.3277858165092766e-05, 2.950529960799031e-05, 2.729639527387917e-05, 2.506915552658029e-05, 2.3916752979857847e-05, 2.256272273370996e-05, 2.187858262914233e-05, 2.0887062419205904e-05, 3.802051651291549e-05, 3.640782961156219e-05, 3.315599315101281e-05, 3.0232957215048373e-05, 5.959425470791757e-05, 5.0155314966104925e-05, 3.854122769553214e-05, 3.1676045182393864e-05, 3.018258576048538e-05, 2.5247787561966106e-05, 2.095119998557493e-05, 1.8922382878372446e-05, 1.1635425835265778e-05, 9.173507351079024e-06, 8.420928679697681e-06, 8.064833309617825e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001051802182701067, "first_to_last_ratio": 1.2988284379869668, "imbalance_abs_log_ratio": 0.26146265659274887, "last_quartile_mean": 8.098084026642027e-05, "mean": 0.00011297554704015056, "normalized": [ 0.8108794110949893, 0.8285381181909731, 0.8190502123585883, 0.796943644872694, 1.0905965877081263, 1.1190586951317285, 1.0312475376881012, 0.951684658247413, 1.0315797924663694, 0.9117273340922073, 0.8439995901115003, 0.7977917028142015, 0.6929493161394222, 0.6545568546712421, 0.6381324299605099, 0.620599759422206, 1.4094209052378723, 1.3397655167973181, 1.2663323805793527, 1.1428262144942543, 2.153338001318954, 2.00370497849062, 1.7845980547125555, 1.5262805588928774, 1.2411593314085447, 1.074164756659053, 0.8778398584235759, 0.7656336393496486, 0.5727781276048317, 0.4449834340085659, 0.3922665989654889, 0.36557199808621427 ], "peak_layer": 21, "peak_normalized": 2.153338001318954, "population_cv": 0.42307789431213444, "rest_mean": 9.745571568836803e-05, "spike_contrast": 2.019200566635041, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019678263633977623, "top_five_layers": [ 21, 22, 23, 24, 17 ], "top_five_spike_overlap": 4 }, "values": [ 9.160954505205154e-05, 9.360454714624211e-05, 9.2532645794563e-05, 9.003514423966408e-05, 0.0001232107460964471, 0.0001264262682525441, 0.0001165057547041215, 0.00010751709487522021, 0.00011654329136945307, 0.00010300289432052523, 9.535131539450958e-05, 9.013095404952765e-05, 7.828632806194946e-05, 7.394891872536391e-05, 7.209336035884917e-05, 7.011259731370956e-05, 0.00015923009777907282, 0.00015136074216570705, 0.0001430645934306085, 0.0001291114167543128, 0.0002432745386613533, 0.0002263696660520509, 0.0002016159414779395, 0.00017243238107766956, 0.00014022065442986786, 0.00012135435099480674, 9.917443821905181e-05, 8.649787923786789e-05, 6.470992229878902e-05, 5.0272246880922467e-05, 4.4316533603705466e-05, 4.130069646635093e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "4fbc436930b501f120196fb3bf6ab490fc20b93e2a3a7db48af6d653a1fa07c9", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16142085194587708, 0.2773628532886505, 0.3768865168094635, 0.40868353843688965, 0.14338424801826477, 0.16827327013015747, 0.18227216601371765, 0.19825012981891632, 0.06985136866569519, 0.1086883395910263, 0.14860449731349945, 0.21192558109760284, 0.08672910183668137, 0.13778631389141083, 0.18114051222801208, 0.23228809237480164, 0.062035780400037766, 0.0909910649061203, 0.12021689862012863, 0.1606491208076477, 0.061457011848688126, 0.092374786734581, 0.12395651638507843, 0.15162576735019684, 0.06645520776510239, 0.10725662857294083, 0.14660172164440155, 0.20186874270439148, 0.07922796159982681, 0.17203646898269653, 0.2743971645832062, 0.41855114698410034 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3263900200399803e-05, "first_to_last_ratio": 1.6142150262450288, "imbalance_abs_log_ratio": 0.478848786652461, "last_quartile_mean": 8.21693515717925e-06, "mean": 1.2790936466444691e-05, "normalized": [ 0.9316267643378715, 0.9631635128907836, 0.9539325686606005, 0.9083160383931895, 1.2160436167127753, 1.219552487596532, 1.0980943454698229, 1.005082460607795, 1.044869976177374, 0.9611479101725001, 0.8963889491080845, 0.8477546675806928, 0.7494694506413357, 0.7060841834105682, 0.692386873457513, 0.6719497697041285, 1.4680346994465847, 1.3664213595724557, 1.2391345582710243, 1.1215611460596746, 2.127901945162036, 1.830484252606031, 1.5253501442672783, 1.3160248210947718, 1.1151359191068313, 0.8926157823445571, 0.7767052939266422, 0.6977092677239847, 0.46888414192457656, 0.41063288917446084, 0.3886900028421406, 0.3888502015553834 ], "peak_layer": 21, "peak_normalized": 2.127901945162036, "population_cv": 0.3879272801672154, "rest_mean": 1.1410037822551439e-05, "spike_contrast": 1.774559336118009, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0247789143468255e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.1916378753085155e-05, 1.2319763300183695e-05, 1.220169087901013e-05, 1.1618212738540024e-05, 1.5554336641798727e-05, 1.5599218386341818e-05, 1.404565500706667e-05, 1.2855945897172205e-05, 1.336486548098037e-05, 1.2293981853872538e-05, 1.1465654097264633e-05, 1.0843576092156582e-05, 9.58641612669453e-06, 9.031477929966059e-06, 8.856276508595329e-06, 8.594866812927648e-06, 1.8777538571157493e-05, 1.7477808796684258e-05, 1.584969140822068e-05, 1.4345817362482194e-05, 2.7217858587391675e-05, 2.341360777791124e-05, 1.9510656784405e-05, 1.6833189874887466e-05, 1.4263632692745887e-05, 1.1417391760915052e-05, 9.934788067766931e-06, 8.924354915507138e-06, 5.997467269480694e-06, 5.252379196463153e-06, 4.971709131496027e-06, 4.9737582230591215e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.259535555524053e-05, "first_to_last_ratio": 1.59358297228639, "imbalance_abs_log_ratio": 0.46598492272985975, "last_quartile_mean": 3.300446633147658e-05, "mean": 5.106260448428657e-05, "normalized": [ 0.921606350307445, 0.9505980521014451, 0.9422292752458163, 0.8985273805847644, 1.2178505351666271, 1.2132371065714032, 1.0942079557361923, 1.0018801076079868, 1.0531977858451338, 0.9641972432463806, 0.8991742893725557, 0.8499991046449304, 0.7546616344988103, 0.712299296999438, 0.6948617566317875, 0.6737987484795189, 1.4721176067352022, 1.3719352314903852, 1.2436616258358075, 1.111210257581308, 2.1130451955668876, 1.8242133884920444, 1.531138344238427, 1.319527925169485, 1.1232592899812257, 0.8974371112918603, 0.7821448561426008, 0.7035301918972675, 0.4712647874324093, 0.41227521981846393, 0.3905171409466258, 0.3903952043397641 ], "peak_layer": 21, "peak_normalized": 2.1130451955668876, "population_cv": 0.3859968358313239, "rest_mean": 4.555695098436748e-05, "spike_contrast": 1.7734534826874866, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.079313338384963e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.705962055595592e-05, 4.8540012357989326e-05, 4.8112680815393105e-05, 4.5881148253101856e-05, 6.218662019819021e-05, 6.19510465185158e-05, 5.5873108067316934e-05, 5.11586076754611e-05, 5.377902198233642e-05, 4.923442247672938e-05, 4.591418110067025e-05, 4.340316809248179e-05, 3.853498856187798e-05, 3.637185727711767e-05, 3.548145105014555e-05, 3.440591899561696e-05, 7.517015910707414e-05, 7.005458610365167e-05, 6.350460171233863e-05, 5.674128988175653e-05, 0.00010789759107865393, 9.314908675150946e-05, 7.818391168257222e-05, 6.73785325489007e-05, 5.735654485761188e-05, 4.582547626341693e-05, 3.993835343862884e-05, 3.59240839316044e-05, 2.40640074480325e-05, 2.1051846488262527e-05, 1.9940822312491946e-05, 1.993459591176361e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04664208646863699, "first_to_last_ratio": 1.5935829291342798, "imbalance_abs_log_ratio": 0.46598489565118767, "last_quartile_mean": 0.029268691083416343, "mean": 0.04528282774845138, "normalized": [ 0.9216063328612899, 0.9505981078752931, 0.9422292327337689, 0.8985273464900081, 1.2178505932652952, 1.2132370501116239, 1.0942079658156032, 1.0018800915901536, 1.053197779258807, 0.9641972831787459, 0.8991742955513574, 0.8499990919117887, 0.7546616376365752, 0.7122993077974369, 0.6948617929269221, 0.6737987636701324, 1.4721176164746403, 1.3719352014835002, 1.243661590239114, 1.1112102873322856, 2.1130452142435634, 1.8242133908087463, 1.531138226506127, 1.3195278850867733, 1.123259302264044, 0.8974371418917119, 0.7821448794692506, 0.7035302028513013, 0.47126479509418523, 0.4122752409211241, 0.39051713630582213, 0.39039521635300933 ], "peak_layer": 21, "peak_normalized": 2.1130452142435634, "population_cv": 0.3859968269414632, "rest_mean": 0.04040035943466204, "spike_contrast": 1.7734534456008406, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07164815664291382, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.04173294082283974, 0.0430457703769207, 0.042666804045438766, 0.04068785905838013, 0.055147718638181686, 0.05493880435824394, 0.04954883083701134, 0.04536796361207962, 0.04769177362322807, 0.04366157948970795, 0.04071715474128723, 0.03849036246538162, 0.03417321294546127, 0.03225492686033249, 0.031465306878089905, 0.030511513352394104, 0.06666164845228195, 0.062125105410814285, 0.05631651356816292, 0.05031874403357506, 0.09568466246128082, 0.08260554075241089, 0.06933426856994629, 0.059751953929662704, 0.05086435750126839, 0.04063849151134491, 0.03541773185133934, 0.03185783699154854, 0.021340202540159225, 0.018668988719582558, 0.017683720216155052, 0.01767819933593273 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.026510429686823e-06, "first_to_last_ratio": 0.915629822205116, "imbalance_abs_log_ratio": 0.0881431202184329, "last_quartile_mean": 2.2132420554044074e-06, "mean": 2.6874848515490157e-06, "normalized": [ 0.8352326104505317, 0.7929193788577268, 0.7729974408049657, 0.7742255616893798, 0.734464344312438, 0.7231120097580448, 0.6957460905757977, 0.7037397057177938, 0.7902611858247006, 0.7709003457125179, 0.755409152404643, 0.7733638634799724, 0.646281017658817, 0.6542165517187554, 0.6700454867303076, 0.7131122495064298, 1.404762079762564, 1.37930285294127, 1.346554090752029, 1.2824702252229325, 2.382204590992269, 2.173774134622282, 1.9010011763534234, 1.7356112976249671, 1.4638471650261105, 1.1467541532177463, 1.0089145048692982, 0.9362415767757399, 0.5805525649285846, 0.49587787710091685, 0.4753518619529011, 0.48075285265414314 ], "peak_layer": 21, "peak_normalized": 2.382204590992269, "population_cv": 0.4843847200620537, "rest_mean": 2.2239993861014092e-06, "spike_contrast": 2.3337714917558996, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.190306364966091e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.244674988105544e-06, 2.1309588191797957e-06, 2.0774189124495024e-06, 2.0807194687222363e-06, 1.9738617993425578e-06, 1.9433525721979095e-06, 1.8698070789469057e-06, 1.8912897985501331e-06, 2.1238149656710448e-06, 2.071783001156291e-06, 2.0301506538089598e-06, 2.078403667837847e-06, 1.7368704448017525e-06, 1.7581970723767881e-06, 1.8007370954364887e-06, 1.916478368002572e-06, 3.7752768093923805e-06, 3.706855522978003e-06, 3.618843720687437e-06, 3.4466193028492853e-06, 6.4021387515822425e-06, 5.841985057486454e-06, 5.108911864226684e-06, 4.664429070544429e-06, 3.934067080990644e-06, 3.0818844152236125e-06, 2.7114424483443145e-06, 2.516135054975166e-06, 1.5602262237734976e-06, 1.3326642829269986e-06, 1.2775009281540406e-06, 1.2920160088469856e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.0595810332888504e-05, "first_to_last_ratio": 1.8428634767077197, "imbalance_abs_log_ratio": 0.6113205992795879, "last_quartile_mean": 2.2028658577255555e-05, "mean": 3.7230811983590684e-05, "normalized": [ 1.000742073656224, 1.0297534180314578, 1.0169311659883222, 0.9702710746421311, 1.271580024525988, 1.257182912808566, 1.1379498051373607, 1.0386461914531528, 1.1094287905906044, 1.0094888620711941, 0.9345745159294909, 0.8772057305692527, 0.8023970134719064, 0.7583468682633459, 0.7417669191528679, 0.7178190266273485, 1.4137682485056282, 1.3271728543994634, 1.203515443425389, 1.0853495701875495, 2.130407864804216, 1.78105851286132, 1.4345507874114394, 1.2166669923043094, 1.0512864996659526, 0.8337322945371913, 0.7246595088199664, 0.6567104258973786, 0.4057836476031255, 0.3620542980954201, 0.34956534917004795, 0.3496333093923887 ], "peak_layer": 21, "peak_normalized": 2.130407864804216, "population_cv": 0.38398827396059443, "rest_mean": 3.362635827754814e-05, "spike_contrast": 1.686024443333039, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.6694861996220425e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 3.7258439988363534e-05, 3.8338555896189064e-05, 3.7861173041164875e-05, 3.612397995311767e-05, 4.734195681521669e-05, 4.6805940655758604e-05, 4.236679524183273e-05, 3.8669641071464866e-05, 4.130493471166119e-05, 3.758409002330154e-05, 3.479496808722615e-05, 3.265908162575215e-05, 2.9873892344767228e-05, 2.823386967065744e-05, 2.7616584702627733e-05, 2.672498521860689e-05, 5.2635739848483354e-05, 4.94117230118718e-05, 4.480785719351843e-05, 4.0408445784123614e-05, 7.931681466288865e-05, 6.631025462411344e-05, 5.340949064702727e-05, 4.529750003712252e-05, 3.914025000995025e-05, 3.104053030256182e-05, 2.6979661924997345e-05, 2.4449862394249067e-05, 1.5107654689927585e-05, 1.347957550024148e-05, 1.3014601790928282e-05, 1.3017132005188614e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.132075494249875e-05, "first_to_last_ratio": 2.1168727266692042, "imbalance_abs_log_ratio": 0.749939870680473, "last_quartile_mean": 1.4795766674069455e-05, "mean": 2.6930997876206675e-05, "normalized": [ 1.0946769735958595, 1.1271800223787605, 1.106456131482209, 1.0633216760799757, 1.3525082558946082, 1.3082808262675252, 1.181611989443304, 1.0699654356061856, 1.1577934996059576, 1.051929807416546, 0.9684829611938787, 0.9018017398021899, 0.8575415520179537, 0.8139981937633387, 0.7942835242774724, 0.7646872384868243, 1.3514805278136963, 1.2804770533277787, 1.1546995085484912, 1.0491442158344637, 2.1271928786890193, 1.6882992242363928, 1.2773347017743915, 1.0616887663741317, 0.9776669976077785, 0.7923208716371506, 0.7110474784846503, 0.6578201335496455, 0.3425598167485049, 0.3136933962077473, 0.3002437073917716, 0.29981089446179765 ], "peak_layer": 21, "peak_normalized": 2.1271928786890193, "population_cv": 0.3819948907959122, "rest_mean": 2.480426438624919e-05, "spike_contrast": 1.5487400925815615, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.84153587219771e-05, "top_five_layers": [ 21, 22, 5, 17, 6 ], "top_five_spike_overlap": 2 }, "values": [ 2.948074325104244e-05, 3.0356082788784988e-05, 2.9797967727063224e-05, 2.8636313800234348e-05, 3.642439696704969e-05, 3.523330815369263e-05, 3.182198997819796e-05, 2.8815236873924732e-05, 3.118053427897394e-05, 2.8329519409453496e-05, 2.6082212571054697e-05, 2.428642073937226e-05, 2.3094449716154486e-05, 2.1921783627476543e-05, 2.139084790542256e-05, 2.0593790395651013e-05, 3.6396719224285334e-05, 3.448452480370179e-05, 3.109721001237631e-05, 2.8254500648472458e-05, 5.7287426898255944e-05, 4.546758282231167e-05, 3.439989814069122e-05, 2.8592337912414223e-05, 2.632954783621244e-05, 2.1337991711334325e-05, 1.9149218132952228e-05, 1.7715752619551495e-05, 9.225477697327733e-06, 8.448076187050901e-06, 8.085862646112218e-06, 8.074206562014297e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001040923161781393, "first_to_last_ratio": 1.4801753983355805, "imbalance_abs_log_ratio": 0.3921605931428911, "last_quartile_mean": 7.032431176412501e-05, "mean": 0.00010517841462842625, "normalized": [ 0.8617262734887446, 0.8795612163230485, 0.8836052514767463, 0.8537133297175012, 1.173918399219713, 1.1881939629889764, 1.0876047193780078, 0.9890668591031853, 1.0219410138030538, 0.9359575171630558, 0.8633693032863031, 0.8295890644515174, 0.71591189277355, 0.6802765265722901, 0.6658347297769002, 0.6504541971660552, 1.4527609839189186, 1.3426235449558381, 1.2433488773729529, 1.1347940072566047, 2.180691877285598, 1.925348956107517, 1.6618204246441632, 1.428933143337476, 1.1531390634220187, 0.9405881065525119, 0.8279860191271594, 0.7404700379978636, 0.4795815886023818, 0.4212812520523834, 0.3924877272922389, 0.3934201333857252 ], "peak_layer": 21, "peak_normalized": 2.180691877285598, "population_cv": 0.4088800352175212, "rest_mean": 9.212875940849901e-05, "spike_contrast": 1.9065333555314292, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017564655281603335, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 9.063500328920782e-05, 9.251085430150852e-05, 9.293619950767606e-05, 8.979221456684172e-05, 0.0001234708761330694, 0.00012497235729824752, 0.0001143925401265733, 0.00010402848420199007, 0.00010748613567557186, 9.844252781476825e-05, 9.080781455850229e-05, 8.725486259208992e-05, 7.529847789555788e-05, 7.155040657380596e-05, 7.003144128248096e-05, 6.841374124633148e-05, 0.00015279909712262452, 0.00014121501590125263, 0.00013077346375212073, 0.00011935583461308852, 0.00022936171444598585, 0.00020250515080988407, 0.0001747876376612112, 0.0001502929226262495, 0.00012128533853683621, 9.892956586554646e-05, 8.708625682629645e-05, 7.788146467646584e-05, 5.044163117418066e-05, 4.430969420354813e-05, 4.1281236917711794e-05, 4.1379305912414566e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e4d8ebf5b6b3ad968bf338e0327e2eabdbae498ab18a1157cbd942424ccea56f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059127259999513626, 0.0761958509683609, 0.13674014806747437, 0.18700072169303894, 0.2049511820077896, 0.14231130480766296, 0.13109618425369263, 0.13412337005138397, 0.13132649660110474, 0.09715579450130463, 0.09982985258102417, 0.10234102606773376, 0.11231652647256851, 0.08827291429042816, 0.08991213887929916, 0.1065397635102272, 0.10877490043640137, 0.07123783230781555, 0.07851143926382065, 0.08676032721996307, 0.10016339272260666, 0.06517573446035385, 0.07041674107313156, 0.07159053534269333, 0.07631570100784302, 0.06180137023329735, 0.06370948255062103, 0.07006713002920151, 0.0730312243103981, 0.06770572811365128, 0.07336394488811493, 0.08089767396450043 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.732241587315002e-06, "first_to_last_ratio": 1.4278944469783923, "imbalance_abs_log_ratio": 0.356200944503622, "last_quartile_mean": 4.7148033957000735e-06, "mean": 8.330031846526254e-06, "normalized": [ 0.5885835786473461, 0.7084430032945174, 0.4907562240746722, 0.6082386339015993, 0.45167837229458263, 1.0561301434994, 1.3931893382898604, 1.1684941266915543, 1.5667867561754885, 1.2816056832829195, 0.9678205108695733, 0.9346667715729674, 0.48015470085429257, 0.7950059323694191, 0.968685455585295, 0.9109115267981576, 0.9428117950257354, 1.5508886748757895, 1.3922681645248334, 1.687647767587324, 1.8711243564793436, 2.3658432545646693, 2.1045605102704337, 1.1856995606832565, 1.1632563088618004, 0.9930387481612112, 0.515645548723529, 0.5621148182174204, 0.3607436892275036, 0.4577143696679217, 0.23690245631552195, 0.23858921861206098 ], "peak_layer": 22, "peak_normalized": 2.3658432545646693, "population_cv": 0.5306581723841309, "rest_mean": 7.191444840120514e-06, "spike_contrast": 2.0132813367827516, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4478401681117248e-05, "top_five_layers": [ 22, 23, 21, 20, 9 ], "top_five_spike_overlap": 3 }, "values": [ 4.9029199544747826e-06, 5.901352778892033e-06, 4.088014975422993e-06, 5.066647190687945e-06, 3.7624952256010147e-06, 8.797597729426343e-06, 1.1605311556195375e-05, 9.73359328781953e-06, 1.3051383575657383e-05, 1.0675816156435758e-05, 8.061975677264854e-06, 7.785803973092698e-06, 3.999703949375544e-06, 6.622424734814558e-06, 8.0691806942923e-06, 7.587922027596505e-06, 7.853652277844958e-06, 1.2918952052132227e-05, 1.1597638149396516e-05, 1.4058159649721347e-05, 1.5586525478283875e-05, 1.9707549654413015e-05, 1.7531056073494256e-05, 9.876915100903716e-06, 9.689962098491378e-06, 8.272044397017453e-06, 4.295343842386501e-06, 4.682434337155428e-06, 3.005006419698475e-06, 3.8127752759464784e-06, 1.9734050056285923e-06, 1.9874557892762823e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.6136923338526685e-05, "first_to_last_ratio": 1.4118866191026362, "imbalance_abs_log_ratio": 0.3449268377610348, "last_quartile_mean": 1.8512055419250828e-05, "mean": 3.305522808716432e-05, "normalized": [ 0.5803678005983632, 0.6305039863571337, 0.4562023660826862, 0.6002682493018332, 0.4447469941024251, 1.006280776549637, 1.3680201611097922, 1.2392470554464203, 1.559214741299254, 1.3270070893842953, 0.9349629066298444, 0.9435481692108943, 0.505187459486473, 0.8266440594730092, 0.9312125841240197, 0.8763433716202387, 1.000973249174259, 1.52217970123748, 1.4468612378475991, 1.722721252602393, 1.878854565585447, 2.4072912511567135, 2.1076999580640594, 1.2033880908130195, 1.1649327575261703, 1.0028277195998705, 0.5138852559553679, 0.5315879646037021, 0.35380571795268584, 0.44968368242147705, 0.22921632783533166, 0.2343334968481047 ], "peak_layer": 22, "peak_normalized": 2.4072912511567135, "population_cv": 0.5417197270843308, "rest_mean": 2.8449328982349942e-05, "spike_contrast": 2.036149368904277, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.792708325316198e-05, "top_five_layers": [ 22, 23, 21, 20, 9 ], "top_five_spike_overlap": 3 }, "values": [ 1.91841900232248e-05, 2.0841453078901395e-05, 1.507987326476723e-05, 1.9842003894154914e-05, 1.4701213331136387e-05, 3.326284058857709e-05, 4.522021845332347e-05, 4.0963594074128196e-05, 5.1540198910515755e-05, 4.3864522012881935e-05, 3.0905412131687626e-05, 3.118919994449243e-05, 1.6699086700100452e-05, 2.7324907932779752e-05, 3.078144436585717e-05, 2.8967730031581596e-05, 3.3087399060605094e-05, 5.031599721405655e-05, 4.78263282275293e-05, 5.694494393537752e-05, 6.210596620803699e-05, 7.957356137922034e-05, 6.967050285311416e-05, 3.9778267819201574e-05, 3.850711800623685e-05, 3.314869900350459e-05, 1.6986594346235506e-05, 1.7571761418366805e-05, 1.169512870546896e-05, 1.486439668951789e-05, 7.576797997899121e-06, 7.745947186776903e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.023178485920652747, "first_to_last_ratio": 1.4118866250838005, "imbalance_abs_log_ratio": 0.3449268419973269, "last_quartile_mean": 0.016416676458902657, "mean": 0.029313707374967635, "normalized": [ 0.5803678083378483, 0.6305039884996685, 0.456202372531724, 0.600268267835863, 0.4447469968655515, 1.006280799172435, 1.3680202139428703, 1.2392470777204811, 1.559214634558375, 1.3270070448652505, 0.9349629030577731, 0.9435480965148331, 0.5051874298044523, 0.826644056489135, 0.9312126036518835, 0.8763433869124639, 1.000973281360552, 1.5221796892520219, 1.4468612026594825, 1.7227212328102954, 1.8788545699778456, 2.407291269948361, 2.1077000369999817, 1.203388036597476, 1.1649328054213401, 1.00282774821406, 0.513885282837708, 0.5315879274732056, 0.35380571518519355, 0.44968368484851295, 0.2292163286219473, 0.2343335070314083 ], "peak_layer": 22, "peak_normalized": 2.407291269948361, "population_cv": 0.5417197285436497, "rest_mean": 0.0252291498084863, "spike_contrast": 2.0361493995603235, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.05137031823396683, "top_five_layers": [ 22, 23, 21, 20, 9 ], "top_five_spike_overlap": 3 }, "values": [ 0.017012732103466988, 0.018482409417629242, 0.01337298285216093, 0.017596088349819183, 0.013037183322012424, 0.029497820883989334, 0.04010174423456192, 0.036326926201581955, 0.045706361532211304, 0.0388994961977005, 0.02740722894668579, 0.027658892795443535, 0.014808916486799717, 0.02423200197517872, 0.02729729376733303, 0.02568887360394001, 0.02934223785996437, 0.044620729982852936, 0.04241286590695381, 0.05049934610724449, 0.05507619306445122, 0.07056663185358047, 0.06178450211882591, 0.035275764763355255, 0.03414849936962128, 0.029396599158644676, 0.015063882805407047, 0.015582812950015068, 0.010371357202529907, 0.013181895948946476, 0.006719180382788181, 0.006869183853268623 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.833606326329118e-06, "first_to_last_ratio": 0.6149451250612414, "imbalance_abs_log_ratio": 0.48622224269933, "last_quartile_mean": 2.9817397546594293e-06, "mean": 2.912170225854993e-06, "normalized": [ 2.082671717873138, 1.0258662958379037, 0.2962831198895636, 0.2730480128818211, 0.2012400182045682, 0.2720270773081725, 0.42178614984566853, 0.4641631742428811, 0.5401984753999569, 0.6459927680651849, 0.5728072013454448, 0.7481885282952891, 0.4630758731723221, 0.5702400277708374, 0.578231799947324, 0.5768551843487628, 0.8350551079719575, 1.4095672389330323, 1.4695978758937063, 1.4976484633572726, 1.689917115612428, 2.7658156568776397, 2.289043897069376, 2.1195653126073872, 1.6714052019812762, 1.6382121473547555, 1.1947076934524972, 1.295704736654288, 0.8014556622731791, 0.6524420889778894, 0.48392041690429416, 0.4532659596501817 ], "peak_layer": 22, "peak_normalized": 2.7658156568776397, "population_cv": 0.6680125549192626, "rest_mean": 2.315094739649929e-06, "spike_contrast": 2.6505947019214586, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.13637785136234e-06, "top_five_layers": [ 22, 23, 24, 1, 21 ], "top_five_spike_overlap": 4 }, "values": [ 6.065094567020424e-06, 2.9874972824472934e-06, 8.628268801658123e-07, 7.9516229334331e-07, 5.860451892658602e-07, 7.921891551632143e-07, 1.2283130672585685e-06, 1.3517221759684617e-06, 1.5731499161120155e-06, 1.8812409052770818e-06, 1.6681120769135305e-06, 2.178852355427807e-06, 1.3485557701642392e-06, 1.660636030464957e-06, 1.6839094314491376e-06, 1.6799004924905603e-06, 2.431822622384061e-06, 4.104899744561408e-06, 4.279719178157393e-06, 4.361407263786532e-06, 4.921326308249263e-06, 8.054526006162632e-06, 6.6660854827205185e-06, 6.172534995130263e-06, 4.867416464549024e-06, 4.7707526391604915e-06, 3.4791921734722564e-06, 3.773312755583902e-06, 2.333975317014847e-06, 1.9000224256160436e-06, 1.4092586297920207e-06, 1.3199876320868498e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.8523472363085602e-05, "first_to_last_ratio": 1.6367299197356435, "imbalance_abs_log_ratio": 0.49270029988784414, "last_quartile_mean": 1.1317366499952186e-05, "mean": 2.2688649906399405e-05, "normalized": [ 0.5580404711156489, 0.6466177790146251, 0.5012363612452404, 0.6198604481052087, 0.4857210377349011, 1.0577981440873514, 1.4196693381283139, 1.2424181153378644, 1.3858726044228813, 1.3021505795717108, 0.9395686627187273, 0.9061129797569311, 0.5424003201469547, 0.8400230538707855, 0.9733318846209535, 0.8833365784843897, 1.0149403960317096, 1.6494378056474266, 1.491698228985569, 1.6819650994199165, 1.9383906831259385, 2.5307914955464006, 2.1573696327610388, 1.2407538293982432, 1.0963126662413099, 0.9513736363429608, 0.49399328204720055, 0.41259651935941966, 0.32015875996296644, 0.33597611060399785, 0.19288429859973458, 0.1871991975636804 ], "peak_layer": 22, "peak_normalized": 2.5307914955464006, "population_cv": 0.5672379252078826, "rest_mean": 1.935794071633373e-05, "spike_contrast": 2.1011780193351854, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.067447953275405e-05, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.2661184882745147e-05, 1.4670884411316365e-05, 1.1372376320650801e-05, 1.4063796697882935e-05, 1.1020354577340186e-05, 2.4000011762836948e-05, 3.221038059564307e-05, 2.8188789656269364e-05, 3.14435783366207e-05, 2.9544038625317626e-05, 2.1317544451449066e-05, 2.055848017334938e-05, 1.230633097293321e-05, 1.905898898257874e-05, 2.2083586372900754e-05, 2.004171437874902e-05, 2.3027627321425825e-05, 3.742351691471413e-05, 3.3844618883449584e-05, 3.816151729552075e-05, 4.3979467591270804e-05, 5.742024222854525e-05, 4.894780431641266e-05, 2.8151029255241156e-05, 2.4873854272300377e-05, 2.158538336516358e-05, 1.1208040632482152e-05, 9.361257980344817e-06, 7.263970019266708e-06, 7.622844350407831e-06, 4.376284323370783e-06, 4.247297056281241e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.2930837215208157e-05, "first_to_last_ratio": 1.8364715924470074, "imbalance_abs_log_ratio": 0.6078461178470359, "last_quartile_mean": 7.041131084406516e-06, "mean": 1.536685377345748e-05, "normalized": [ 0.5313414842881189, 0.6754785013800788, 0.5526059414596252, 0.6240789848356078, 0.5290051964319934, 1.0858225994860433, 1.4732941115929494, 1.2601804357195125, 1.2171383832168858, 1.2767137827335708, 0.9427961388447176, 0.8464954353259013, 0.5939861247138315, 0.8658328722737771, 1.029153088938301, 0.8803697744896689, 1.0401785162972257, 1.7727172708005683, 1.5249027896433116, 1.5954681052275985, 1.9631529625513957, 2.591841549742642, 2.193273942807079, 1.268551867535944, 1.0451422845983234, 0.9043570007378636, 0.4862798750552231, 0.35652120226339873, 0.2993186376401588, 0.2463406294000189, 0.17093996322188035, 0.15672054674678554 ], "peak_layer": 22, "peak_normalized": 2.591841549742642, "population_cv": 0.582798597235848, "rest_mean": 1.3055017276470877e-05, "spike_contrast": 2.133338491047477, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7850770857185124e-05, "top_five_layers": [ 22, 23, 21, 18, 20 ], "top_five_spike_overlap": 3 }, "values": [ 8.165046892827377e-06, 1.0379979357821867e-05, 8.491814696753863e-06, 9.590130503056571e-06, 8.129145498969592e-06, 1.668567711021751e-05, 2.2639895178144798e-05, 1.936500848387368e-05, 1.8703587556956336e-05, 1.9619074009824544e-05, 1.4487810403807089e-05, 1.3007971574552357e-05, 9.127697921940126e-06, 1.3305127140483819e-05, 1.581484502821695e-05, 1.3528513591154478e-05, 1.5984271158231422e-05, 2.7241087082074955e-05, 2.343295818718616e-05, 2.4517325073247775e-05, 3.0167484510457143e-05, 3.98284500988666e-05, 3.370371996425092e-05, 1.949365105247125e-05, 1.6060548659879714e-05, 1.3897121789341327e-05, 7.472591732948786e-06, 5.478609182318905e-06, 4.5995857362868264e-06, 3.785480430451571e-06, 2.6268094188708346e-06, 2.4083017251541605e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.1106853561577736e-05, "first_to_last_ratio": 1.4788913820367076, "imbalance_abs_log_ratio": 0.3912927408943512, "last_quartile_mean": 3.4557543699520465e-05, "mean": 6.540310639024938e-05, "normalized": [ 0.5486856091946496, 0.6274437757279189, 0.43571932821426484, 0.625224380313627, 0.45136709492800875, 1.0286093422994689, 1.3184260053458061, 1.2158304195841385, 1.5307439392265534, 1.3325715083590115, 0.9176469695695924, 0.9806287946397408, 0.5097569130124642, 0.831748301564442, 0.897166288563406, 0.8855516750577264, 1.004404805032616, 1.5575747054430396, 1.497114703177346, 1.776321321168683, 2.025217116434768, 2.4423949256211186, 2.124634016222874, 1.2081963668463171, 1.151934526222184, 1.004674358671154, 0.4849562539246849, 0.4229542168376202, 0.3250708970312119, 0.41524843702243736, 0.21300096811608038, 0.209182036627046 ], "peak_layer": 22, "peak_normalized": 2.4423949256211186, "population_cv": 0.5647711514503198, "rest_mean": 5.582911638123699e-05, "spike_contrast": 2.097519359598391, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011710265243891627, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 3.588574327295646e-05, 4.1036772017832845e-05, 2.849739757948555e-05, 4.0891616663429886e-05, 2.9520810130634345e-05, 6.72742462484166e-05, 8.622915629530326e-05, 7.951908628456295e-05, 0.0001001154087134637, 8.715431613381952e-05, 6.001696237944998e-05, 6.413616938516498e-05, 3.333968561491929e-05, 5.4398922657128423e-05, 5.867746222065762e-05, 5.7917830417864025e-05, 6.569119432242587e-05, 0.00010187022417085245, 9.791595221031457e-05, 0.00011617693235166371, 0.0001324554905295372, 0.00015974021516740322, 0.00013895766460336745, 7.901979552116245e-05, 7.534009637311101e-05, 6.570882396772504e-05, 3.171764547005296e-05, 2.7662519642035477e-05, 2.1260646462906152e-05, 2.7158537704963237e-05, 1.3930924978922121e-05, 1.368115499644773e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6920ec3aaf9b121de9e3648519271c2cd5d5a0dcd243425164f6c664720841a8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.055979277938604355, 0.07538121938705444, 0.18601374328136444, 0.08912894874811172, 0.12973569333553314, 0.13784728944301605, 0.1306019276380539, 0.13534265756607056, 0.08038423955440521, 0.0852823480963707, 0.08248664438724518, 0.09743349254131317, 0.07765212655067444, 0.07436380535364151, 0.08191535621881485, 0.09093168377876282, 0.06605636328458786, 0.06527329236268997, 0.0699063166975975, 0.08022409677505493, 0.05980068817734718, 0.05788872763514519, 0.0643942728638649, 0.07148924469947815, 0.056988880038261414, 0.05852367728948593, 0.06811124831438065, 0.07283915579319, 0.057786133140325546, 0.07090020179748535, 0.07763758301734924, 0.07564623653888702 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.8137599602805494e-06, "first_to_last_ratio": 0.257680145287283, "imbalance_abs_log_ratio": 1.3560362101310268, "last_quartile_mean": 1.480036405610008e-05, "mean": 1.100234549866741e-05, "normalized": [ 0.8846955474380374, 0.33662903052191306, 0.12213284218224829, 0.434361012681639, 0.26037541556803, 0.1262485959911879, 0.1939520037247958, 0.4146578693220764, 0.7427772495066182, 0.5340713440899699, 0.5540395486368574, 0.4982682858424812, 0.5539022855457759, 0.5320020234280948, 0.4968872232332075, 0.4886950428755402, 1.256050776052354, 1.2616225581229157, 1.3245147622521796, 1.345633356324533, 2.1682720990903794, 2.4117590499712667, 2.234761662567843, 2.0620839500138963, 2.0505140064546468, 1.8952882564355573, 1.5733863445633343, 1.481268881612435, 1.1894375476641659, 0.9329830578726799, 0.8176375882880192, 0.8210907821253209 ], "peak_layer": 22, "peak_normalized": 2.4117590499712667, "population_cv": 0.672071089583683, "rest_mean": 8.586967678807182e-06, "spike_contrast": 2.800218496834122, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4045385725912637e-05, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 9.73372607404599e-06, 3.7037088986835442e-06, 1.3437477264233166e-06, 4.778989932674449e-06, 2.8647402814385714e-06, 1.3890306718167267e-06, 2.1339269551390316e-06, 4.562209142022766e-06, 8.1722919276217e-06, 5.876037448615534e-06, 6.095734534028452e-06, 5.482119831867749e-06, 6.094224318076158e-06, 5.8532700677460525e-06, 5.4669249038852286e-06, 5.376791705202777e-06, 1.3819504601997323e-05, 1.3880807273380924e-05, 1.4572769032383803e-05, 1.4805123100813944e-05, 2.385607876931317e-05, 2.6535006327321753e-05, 2.4587619918747805e-05, 2.2687760065309703e-05, 2.2560463548870757e-05, 2.0852616216870956e-05, 1.731094016577117e-05, 1.6297432011924684e-05, 1.3086602848488837e-05, 1.0265001947118435e-05, 8.995931239041965e-06, 9.033924470713828e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.4820166825302294e-05, "first_to_last_ratio": 0.250213415325602, "imbalance_abs_log_ratio": 1.38544106397906, "last_quartile_mean": 5.9230104852758814e-05, "mean": 4.3850411316270765e-05, "normalized": [ 0.8638548954533332, 0.33176746760552467, 0.11339954764263065, 0.425856726056105, 0.25141584739259687, 0.12462507515024858, 0.1898225194515901, 0.4030258231564756, 0.744960226144929, 0.5352705464819547, 0.5547974738280123, 0.5031678209626799, 0.552108547371514, 0.533071767737807, 0.49794336735911743, 0.49026058404355205, 1.2586675503373956, 1.2670879196781222, 1.333871541046564, 1.341755883919874, 2.1805808903209534, 2.388311265067082, 2.2357839008695124, 2.0727457387753443, 2.055304013291544, 1.9056304238078259, 1.572799386274169, 1.4807257856235785, 1.1970813365071429, 0.9443362704093409, 0.820902093251126, 0.8290677649823542 ], "peak_layer": 22, "peak_normalized": 2.388311265067082, "population_cv": 0.6740429802830727, "rest_mean": 3.421513476361589e-05, "spike_contrast": 2.802295106040794, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.588090470060706e-05, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.788039248320274e-05, 1.4548139915859792e-05, 4.9726168072083965e-06, 1.8673992599360645e-05, 1.1024688319594134e-05, 5.464860805659555e-06, 8.323795555043034e-06, 1.767284811649006e-05, 3.266681233071722e-05, 2.347183362871874e-05, 2.4328097424586304e-05, 2.20641159103252e-05, 2.4210186893469654e-05, 2.337541627639439e-05, 2.1835021470906213e-05, 2.149812826246489e-05, 5.519308979273774e-05, 5.556232645176351e-05, 5.8490815717959777e-05, 5.883654739591293e-05, 9.561936894897372e-05, 0.00010472843132447451, 9.804004366742447e-05, 9.089075319934636e-05, 9.012592636281624e-05, 8.356267790077254e-05, 6.896790000610054e-05, 6.493043474620208e-05, 5.2492508984869346e-05, 4.140953387832269e-05, 3.5996894439449534e-05, 3.6354962503537536e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.013142672483809292, "first_to_last_ratio": 0.25021342279299313, "imbalance_abs_log_ratio": 1.3854410341349725, "last_quartile_mean": 0.05252584908157587, "mean": 0.03888698373339139, "normalized": [ 0.8638549177389022, 0.3317674544941228, 0.11339954573412385, 0.4258567477072783, 0.25141586355000733, 0.12462508052643778, 0.18982251383769766, 0.4030258156098557, 0.7449602264335986, 0.5352705630942872, 0.5547974744780005, 0.5031678339346759, 0.5521085240028383, 0.5330717624323057, 0.4979434011756376, 0.49026060341985517, 1.2586675506734653, 1.2670878962474197, 1.3338714669036535, 1.3417559191804453, 2.180580864665263, 2.3883112699220628, 2.2357840053368334, 2.0727457982123334, 2.0553038796184118, 1.9056304336464245, 1.5727993114276466, 1.4807258688900127, 1.1970813019784332, 0.9443362429514629, 0.8209021215723841, 0.8290677406041228 ], "peak_layer": 22, "peak_normalized": 2.3883112699220628, "population_cv": 0.6740429788021378, "rest_mean": 0.030342324015994866, "spike_contrast": 2.802295109712436, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08502814620733261, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.03359271213412285, 0.012901435606181622, 0.004409766290336847, 0.016560284420847893, 0.009776804596185684, 0.004846293479204178, 0.007381625007838011, 0.015672458335757256, 0.028969256207346916, 0.020815057680010796, 0.02157440036535263, 0.019566679373383522, 0.021469835191965103, 0.02072955295443535, 0.019363516941666603, 0.019064756110310555, 0.04894578456878662, 0.04927322641015053, 0.05187023803591728, 0.05217684060335159, 0.08479621261358261, 0.09287422150373459, 0.0869428962469101, 0.08060283213853836, 0.0799245685338974, 0.07410421967506409, 0.06116142123937607, 0.057580962777137756, 0.04655088111758232, 0.03672238811850548, 0.03192240744829178, 0.032239943742752075 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 8.559198363400355e-07, "first_to_last_ratio": 0.2186950684356118, "imbalance_abs_log_ratio": 1.5200769010021633, "last_quartile_mean": 3.913759201168432e-06, "mean": 2.80401877361669e-06, "normalized": [ 0.9040556019078396, 0.32702629052223714, 0.09351520076589508, 0.43074186706516326, 0.22724111410332884, 0.09525147785722864, 0.14581699925344985, 0.2183317787379819, 0.6348218757245028, 0.43653474839421297, 0.5014505522601324, 0.42281266866764283, 0.5237432404167371, 0.5038080787557457, 0.48028933248389244, 0.4797106038616353, 1.2212117799762032, 1.2800650021631181, 1.3574152409248479, 1.4176280739859934, 2.2743522375387655, 2.3285282750638068, 2.3417080754487287, 2.187797237735209, 2.1078913327288835, 1.993431185351473, 1.6687004494389643, 1.5599769890603215, 1.2136943895243368, 0.9467738344272854, 0.8188081541617154, 0.8568663116927213 ], "peak_layer": 23, "peak_normalized": 2.3417080754487287, "population_cv": 0.7184404459833599, "rest_mean": 2.155950095651928e-06, "spike_contrast": 2.9238105498542586, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.303589634626405e-06, "top_five_layers": [ 23, 22, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.5349888801429188e-06, 9.169878580905788e-07, 2.6221837856610364e-07, 1.2078082818334224e-06, 6.371883500833064e-07, 2.6708693212640355e-07, 4.0887360341912427e-07, 6.122064064584265e-07, 1.780052457434067e-06, 1.2240516298334114e-06, 1.406076762577868e-06, 1.1855746606670436e-06, 1.4685858786833705e-06, 1.4126873111308669e-06, 1.3467403050526627e-06, 1.3451175391310244e-06, 3.4243007576151285e-06, 3.589326297515072e-06, 3.8062178191466955e-06, 3.975055733462796e-06, 6.377326371875824e-06, 6.529236998176202e-06, 6.566193405888043e-06, 6.134624527476262e-06, 5.910566869715694e-06, 5.589618467638502e-06, 4.679067387769464e-06, 4.374204763735179e-06, 3.4032218536594883e-06, 2.654771606103168e-06, 2.295953436259879e-06, 2.4026692244660808e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.0053945828758515e-05, "first_to_last_ratio": 0.27064297247327324, "imbalance_abs_log_ratio": 1.3069547714438037, "last_quartile_mean": 3.714837203006027e-05, "mean": 2.8984760795935927e-05, "normalized": [ 0.8999349409837173, 0.32728285540106666, 0.1058957173747875, 0.44016555637738164, 0.2552979443105837, 0.12918922677367176, 0.20615311725153693, 0.41104115254674733, 0.764240914256914, 0.5920784377209981, 0.6110717682804259, 0.557954391842857, 0.6124804063036768, 0.5885550227171041, 0.5526510071226007, 0.5526852723084109, 1.2329478642521525, 1.2506174039150972, 1.3311309260294062, 1.3670973218339744, 2.302706839732118, 2.480544409221905, 2.19254363978492, 1.9825191370367767, 2.0485913267373626, 1.8175804645452507, 1.4838659208054645, 1.396779884204161, 1.116946905853611, 0.8583881521618855, 0.7577608404635063, 0.773301231849928 ], "peak_layer": 22, "peak_normalized": 2.480544409221905, "population_cv": 0.6636797496656769, "rest_mean": 2.2536289878661154e-05, "spike_contrast": 2.8312780893733493, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.38065037492197e-05, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.608439899631776e-05, 9.486215276410803e-06, 3.069362037422252e-06, 1.2758093362208456e-05, 7.399749847536441e-06, 3.7445188354467973e-06, 5.97529879087233e-06, 1.191392948385328e-05, 2.215134009020403e-05, 1.7161251889774576e-05, 1.771176903275773e-05, 1.6172174582607113e-05, 1.775259806890972e-05, 1.7059126548701897e-05, 1.6018457245081663e-05, 1.6019450413296e-05, 3.573669891920872e-05, 3.624884629971348e-05, 3.858251147903502e-05, 3.962498885812238e-05, 6.674340693280101e-05, 7.189798634499311e-05, 6.355035293381661e-05, 5.746284296037629e-05, 5.9377929574111477e-05, 5.2682134992210194e-05, 4.300949876778759e-05, 4.048533082823269e-05, 3.237443888792768e-05, 2.48801752604777e-05, 2.1963516701362096e-05, 2.2413951228372753e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 6.8028442683498724e-06, "first_to_last_ratio": 0.30342053392291607, "imbalance_abs_log_ratio": 1.1926355349810676, "last_quartile_mean": 2.2420513801080233e-05, "mean": 1.860519103047409e-05, "normalized": [ 0.979119532254684, 0.3415768918027291, 0.09970038372228712, 0.4595782792628581, 0.258771439136117, 0.13567266851847518, 0.22194219475035443, 0.4287767646116532, 0.7944047903163591, 0.6593126061472351, 0.6705300974303611, 0.6146988416484243, 0.681950359220749, 0.661289911896569, 0.6390706540768326, 0.6305979949978427, 1.2424026640613004, 1.27008934410493, 1.3446579537308916, 1.4010843712596734, 2.4186677284743303, 2.542833867301576, 2.061542914299684, 1.8011865606538167, 1.9777885269447963, 1.6774560753368692, 1.3713733102501011, 1.3230990688095643, 1.0599527142158984, 0.7835783704069301, 0.7234063690287212, 0.7238867513273863 ], "peak_layer": 22, "peak_normalized": 2.542833867301576, "population_cv": 0.6397326686677515, "rest_mean": 1.4607128697982128e-05, "spike_contrast": 2.7517199620130204, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.019472762593068e-05, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.821670593926683e-05, 6.355103323585354e-06, 1.854944684964721e-06, 8.550541679142043e-06, 4.814492058358155e-06, 2.5242159154004185e-06, 4.1292769310530275e-06, 7.97747361502843e-06, 1.4780052879359573e-05, 1.2266636986169033e-05, 1.247534055437427e-05, 1.1436589375080075e-05, 1.2687816706602462e-05, 1.2303425137361046e-05, 1.1890031601069495e-05, 1.1732396160368808e-05, 2.3115138901630417e-05, 2.363025487284176e-05, 2.5017618099809624e-05, 2.6067442377097905e-05, 4.4999775127507746e-05, 4.730990985990502e-05, 3.83553997380659e-05, 3.351142004248686e-05, 3.6797133361687884e-05, 3.120939072687179e-05, 2.551466241129674e-05, 2.4616510927444324e-05, 1.97206227312563e-05, 1.457862526876852e-05, 1.3459113688440993e-05, 1.3468051292875316e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.855429124792863e-05, "first_to_last_ratio": 0.22282350811084511, "imbalance_abs_log_ratio": 1.5013752643278568, "last_quartile_mean": 0.00012814757064916193, "mean": 9.17358819947367e-05, "normalized": [ 0.7746039998248817, 0.2866381886199903, 0.1036281000087033, 0.4039563945354074, 0.2464815307667851, 0.1160751938545964, 0.18551560419629914, 0.3732316220987844, 0.7092844774450237, 0.5116442907474149, 0.5377274775333667, 0.4967237837368528, 0.5197471875160191, 0.5027122037293308, 0.4626857797839783, 0.4753195792444178, 1.1837655470220732, 1.1937355792503366, 1.3296888336896793, 1.292665760701516, 2.2124766020724245, 2.4217320353409204, 2.3397097366792714, 2.14490074947905, 2.1260014458866174, 2.0351633744735502, 1.6931959406501869, 1.5581194253201036, 1.1956665626368055, 0.9514093786344858, 0.8264167114021962, 0.7893769031189312 ], "peak_layer": 22, "peak_normalized": 2.4217320353409204, "population_cv": 0.708541458383297, "rest_mean": 7.05183218896208e-05, "spike_contrast": 2.925632672956847, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020631070656236262, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 7.10589811205864e-05, 2.629500704642851e-05, 9.50641515373718e-06, 3.7057296140119433e-05, 2.2611200620303862e-05, 1.0648260285961442e-05, 1.701843757473398e-05, 3.423873204155825e-05, 6.506683712359518e-05, 4.693614027928561e-05, 4.932890442432836e-05, 4.556739440886304e-05, 4.767946666106582e-05, 4.611674739862792e-05, 4.2444888094905764e-05, 4.3603860831353813e-05, 0.00010859377653105184, 0.00010950838623102754, 0.0001219801779370755, 0.00011858383368235081, 0.00020296349248383194, 0.00022215972421690822, 0.0002146353363059461, 0.00019676436204463243, 0.00019503061776049435, 0.00018669750716071576, 0.00015532682300545275, 0.00014293545973487198, 0.00010968552669510245, 8.727837848709896e-05, 7.581206591567025e-05, 7.241418643388897e-05 ] } } } } } }, "step": 4000 }, { "forward_identity_gate": { "comparisons": { "detached_learned": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_value_backward": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "modes": { "detached_learned": { "bits_per_byte": 1.8565786854442996, "forward": { "activation_sha256": { "attention_branch_output": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", "mlp_branch_output": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", "post_attention_state": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", "post_mlp_state": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", "pre_attention_input": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", "pre_mlp_input": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f" }, "logits_sha256": "a911233a29684ae3ced35122e9aee12a584fd2d6b7e0272c227ab94eb4e607ec", "loss_nats": 1.2868822813034058, "mixer_summary_sha256": "998a9ce6e3ee4cab0180aeb81cc12c18ccf6b1832b43c8f455133227a7e72a26" }, "loss_nats": 1.2868822813034058, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.21598131954669952, "group": 1, "latest_source_mass": 0.061274416744709015, "layer": 1, "max_source_mass": 0.9387255907058716, "mixer_index": 1, "normalized_entropy": 0.31159517863467795, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9387255907058716, "median": 0.9528161883354187, "p05": 0.8532108068466187, "p95": 0.9815421104431152 }, { "label": "current_group_partial", "mean": 0.061274416744709015, "median": 0.047183819115161896, "p05": 0.018457919359207153, "p95": 0.14678917825222015 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5377621054649353, "group": 1, "latest_source_mass": 0.2917928099632263, "layer": 2, "max_source_mass": 0.7082071900367737, "mixer_index": 2, "normalized_entropy": 0.7758267227322699, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.7082071900367737, "median": 0.7268136143684387, "p05": 0.38468095660209656, "p95": 0.923936128616333 }, { "label": "current_group_partial", "mean": 0.2917928099632263, "median": 0.27318641543388367, "p05": 0.07606390863656998, "p95": 0.6153190732002258 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.43774232268333435, "group": 1, "latest_source_mass": 0.170285165309906, "layer": 2, "max_source_mass": 0.829714834690094, "mixer_index": 3, "normalized_entropy": 0.6315286781224629, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.829714834690094, "median": 0.85542893409729, "p05": 0.687734842300415, "p95": 0.9260365962982178 }, { "label": "current_group_partial", "mean": 0.170285165309906, "median": 0.14457105100154877, "p05": 0.07396337389945984, "p95": 0.31226515769958496 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6749181151390076, "group": 1, "latest_source_mass": 0.45927006006240845, "layer": 3, "max_source_mass": 0.5407299995422363, "mixer_index": 4, "normalized_entropy": 0.9737010177171727, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5407299995422363, "median": 0.5310744643211365, "p05": 0.4232047200202942, "p95": 0.6916534900665283 }, { "label": "current_group_partial", "mean": 0.45927006006240845, "median": 0.4689255356788635, "p05": 0.3083464205265045, "p95": 0.576795220375061 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6663151383399963, "group": 1, "latest_source_mass": 0.49885618686676025, "layer": 3, "max_source_mass": 0.5011438131332397, "mixer_index": 5, "normalized_entropy": 0.9612895457523564, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5011438131332397, "median": 0.4804786741733551, "p05": 0.3554112911224365, "p95": 0.7207092642784119 }, { "label": "current_group_partial", "mean": 0.49885618686676025, "median": 0.5195213556289673, "p05": 0.27929070591926575, "p95": 0.6445887684822083 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6865031719207764, "group": 1, "latest_source_mass": 0.4691343307495117, "layer": 4, "max_source_mass": 0.5308656692504883, "mixer_index": 6, "normalized_entropy": 0.9904147216846476, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5308656692504883, "median": 0.5278033018112183, "p05": 0.45786383748054504, "p95": 0.6093498468399048 }, { "label": "current_group_partial", "mean": 0.4691343307495117, "median": 0.47219666838645935, "p05": 0.3906501829624176, "p95": 0.5421361327171326 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4133813679218292, "group": 1, "latest_source_mass": 0.15208598971366882, "layer": 4, "max_source_mass": 0.8479140400886536, "mixer_index": 7, "normalized_entropy": 0.5963832494967191, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.8479140400886536, "median": 0.8514594435691833, "p05": 0.7530128359794617, "p95": 0.9402707815170288 }, { "label": "current_group_partial", "mean": 0.15208598971366882, "median": 0.14854055643081665, "p05": 0.05972917377948761, "p95": 0.24698717892169952 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6850745677947998, "group": 2, "latest_source_mass": 0.4601486325263977, "layer": 5, "max_source_mass": 0.5398513674736023, "mixer_index": 8, "normalized_entropy": 0.9883536815967077, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5398513674736023, "median": 0.5387102365493774, "p05": 0.46027374267578125, "p95": 0.623892605304718 }, { "label": "completed_group_1", "mean": 0.4601486325263977, "median": 0.46128979325294495, "p05": 0.376107394695282, "p95": 0.5397262573242188 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0074119567871094, "group": 2, "latest_source_mass": 0.4627366364002228, "layer": 5, "max_source_mass": 0.4627366364002228, "mixer_index": 9, "normalized_entropy": 0.9169858804405273, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.32744842767715454, "median": 0.30657994747161865, "p05": 0.16577640175819397, "p95": 0.5453717708587646 }, { "label": "completed_group_1", "mean": 0.20981493592262268, "median": 0.2177819311618805, "p05": 0.14342713356018066, "p95": 0.2433384209871292 }, { "label": "current_group_partial", "mean": 0.4627366364002228, "median": 0.4847465753555298, "p05": 0.27109938859939575, "p95": 0.6081984639167786 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0489801168441772, "group": 2, "latest_source_mass": 0.4763939380645752, "layer": 6, "max_source_mass": 0.4763939380645752, "mixer_index": 10, "normalized_entropy": 0.9548228503031734, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28095996379852295, "median": 0.2765299677848816, "p05": 0.23549969494342804, "p95": 0.3424558639526367 }, { "label": "completed_group_1", "mean": 0.24264612793922424, "median": 0.23937124013900757, "p05": 0.21875721216201782, "p95": 0.2814398407936096 }, { "label": "current_group_partial", "mean": 0.4763939380645752, "median": 0.48882943391799927, "p05": 0.38478636741638184, "p95": 0.528400719165802 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.8527575731277466, "group": 2, "latest_source_mass": 0.6454645395278931, "layer": 6, "max_source_mass": 0.6454645395278931, "mixer_index": 11, "normalized_entropy": 0.7762133938639787, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20522089302539825, "median": 0.1591135561466217, "p05": 0.11052663624286652, "p95": 0.47355735301971436 }, { "label": "completed_group_1", "mean": 0.1493145078420639, "median": 0.15203402936458588, "p05": 0.10004506260156631, "p95": 0.18058395385742188 }, { "label": "current_group_partial", "mean": 0.6454645395278931, "median": 0.68653404712677, "p05": 0.4207671880722046, "p95": 0.7333818674087524 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.049942135810852, "group": 2, "latest_source_mass": 0.4395729899406433, "layer": 7, "max_source_mass": 0.4395729899406433, "mixer_index": 12, "normalized_entropy": 0.9556985177033998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35788866877555847, "median": 0.36233147978782654, "p05": 0.30507102608680725, "p95": 0.3985886573791504 }, { "label": "completed_group_1", "mean": 0.20253834128379822, "median": 0.19928529858589172, "p05": 0.180739626288414, "p95": 0.23214691877365112 }, { "label": "current_group_partial", "mean": 0.4395729899406433, "median": 0.43975552916526794, "p05": 0.3959323763847351, "p95": 0.4790603220462799 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9791408181190491, "group": 2, "latest_source_mass": 0.5214842557907104, "layer": 7, "max_source_mass": 0.5214842557907104, "mixer_index": 13, "normalized_entropy": 0.8912523810434521, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.31465935707092285, "median": 0.28992873430252075, "p05": 0.20534247159957886, "p95": 0.507034420967102 }, { "label": "completed_group_1", "mean": 0.1638563871383667, "median": 0.1667816936969757, "p05": 0.11925681680440903, "p95": 0.19515612721443176 }, { "label": "current_group_partial", "mean": 0.5214842557907104, "median": 0.5455081462860107, "p05": 0.35950881242752075, "p95": 0.6097541451454163 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0485806465148926, "group": 2, "latest_source_mass": 0.4457142949104309, "layer": 8, "max_source_mass": 0.4457142949104309, "mixer_index": 14, "normalized_entropy": 0.9544592367395849, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.346283882856369, "median": 0.353412926197052, "p05": 0.26436588168144226, "p95": 0.4084087312221527 }, { "label": "completed_group_1", "mean": 0.20800183713436127, "median": 0.20826736092567444, "p05": 0.18089595437049866, "p95": 0.23509056866168976 }, { "label": "current_group_partial", "mean": 0.4457142949104309, "median": 0.4416901469230652, "p05": 0.3883572816848755, "p95": 0.5136083364486694 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9801348447799683, "group": 2, "latest_source_mass": 0.543371319770813, "layer": 8, "max_source_mass": 0.543371319770813, "mixer_index": 15, "normalized_entropy": 0.8921571831025336, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2748759388923645, "median": 0.2577607035636902, "p05": 0.17991375923156738, "p95": 0.41774845123291016 }, { "label": "completed_group_1", "mean": 0.1817527413368225, "median": 0.17968356609344482, "p05": 0.130966454744339, "p95": 0.23766928911209106 }, { "label": "current_group_partial", "mean": 0.543371319770813, "median": 0.5535831451416016, "p05": 0.4452384412288666, "p95": 0.6142411828041077 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0688585042953491, "group": 3, "latest_source_mass": 0.36256325244903564, "layer": 9, "max_source_mass": 0.4051227569580078, "mixer_index": 16, "normalized_entropy": 0.9729169383233166, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4051227569580078, "median": 0.4008670449256897, "p05": 0.3286888897418976, "p95": 0.48389261960983276 }, { "label": "completed_group_1", "mean": 0.23231396079063416, "median": 0.23353952169418335, "p05": 0.2008797824382782, "p95": 0.2593720257282257 }, { "label": "completed_group_2", "mean": 0.36256325244903564, "median": 0.3638863265514374, "p05": 0.3081618845462799, "p95": 0.4163197875022888 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2915732860565186, "group": 3, "latest_source_mass": 0.36661261320114136, "layer": 9, "max_source_mass": 0.36661261320114136, "mixer_index": 17, "normalized_entropy": 0.931673187369201, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.26621371507644653, "median": 0.23494771122932434, "p05": 0.16815711557865143, "p95": 0.4694361090660095 }, { "label": "completed_group_1", "mean": 0.1108347550034523, "median": 0.11330658197402954, "p05": 0.06620892137289047, "p95": 0.13892118632793427 }, { "label": "completed_group_2", "mean": 0.2563389539718628, "median": 0.26728466153144836, "p05": 0.16010139882564545, "p95": 0.3091832995414734 }, { "label": "current_group_partial", "mean": 0.36661261320114136, "median": 0.36950358748435974, "p05": 0.29210716485977173, "p95": 0.42882072925567627 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3568191528320312, "group": 3, "latest_source_mass": 0.30908840894699097, "layer": 10, "max_source_mass": 0.30908840894699097, "mixer_index": 18, "normalized_entropy": 0.978738131586968, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.294538676738739, "median": 0.2961598038673401, "p05": 0.2467859834432602, "p95": 0.34424129128456116 }, { "label": "completed_group_1", "mean": 0.1710844486951828, "median": 0.17187124490737915, "p05": 0.1508811116218567, "p95": 0.1898101568222046 }, { "label": "completed_group_2", "mean": 0.2252884805202484, "median": 0.22875282168388367, "p05": 0.18443678319454193, "p95": 0.25181543827056885 }, { "label": "current_group_partial", "mean": 0.30908840894699097, "median": 0.3080518841743469, "p05": 0.2719307243824005, "p95": 0.3494151830673218 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.320021152496338, "group": 3, "latest_source_mass": 0.3312946557998657, "layer": 10, "max_source_mass": 0.3312946557998657, "mixer_index": 19, "normalized_entropy": 0.9521939852875004, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30465462803840637, "median": 0.311603307723999, "p05": 0.21955373883247375, "p95": 0.40013614296913147 }, { "label": "completed_group_1", "mean": 0.12911874055862427, "median": 0.12981033325195312, "p05": 0.09804825484752655, "p95": 0.15542283654212952 }, { "label": "completed_group_2", "mean": 0.23493197560310364, "median": 0.24103470146656036, "p05": 0.16810482740402222, "p95": 0.28502923250198364 }, { "label": "current_group_partial", "mean": 0.3312946557998657, "median": 0.325168639421463, "p05": 0.2647673785686493, "p95": 0.41786038875579834 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3578683137893677, "group": 3, "latest_source_mass": 0.30723780393600464, "layer": 11, "max_source_mass": 0.30723780393600464, "mixer_index": 20, "normalized_entropy": 0.9794949412420898, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29558902978897095, "median": 0.2938143014907837, "p05": 0.24335844814777374, "p95": 0.3542291522026062 }, { "label": "completed_group_1", "mean": 0.17812997102737427, "median": 0.17726755142211914, "p05": 0.15839283168315887, "p95": 0.20142368972301483 }, { "label": "completed_group_2", "mean": 0.21904319524765015, "median": 0.2218058705329895, "p05": 0.18076662719249725, "p95": 0.24698719382286072 }, { "label": "current_group_partial", "mean": 0.30723780393600464, "median": 0.30536961555480957, "p05": 0.26194074749946594, "p95": 0.35952210426330566 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.258887767791748, "group": 3, "latest_source_mass": 0.34303945302963257, "layer": 11, "max_source_mass": 0.35744374990463257, "mixer_index": 21, "normalized_entropy": 0.9080955698144659, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35744374990463257, "median": 0.359974205493927, "p05": 0.17910680174827576, "p95": 0.5207242965698242 }, { "label": "completed_group_1", "mean": 0.12067858874797821, "median": 0.11769672483205795, "p05": 0.08151324093341827, "p95": 0.16476252675056458 }, { "label": "completed_group_2", "mean": 0.17883820831775665, "median": 0.17648206651210785, "p05": 0.0995911955833435, "p95": 0.2739725708961487 }, { "label": "current_group_partial", "mean": 0.34303945302963257, "median": 0.33267199993133545, "p05": 0.22800606489181519, "p95": 0.48691368103027344 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3539760112762451, "group": 3, "latest_source_mass": 0.3175881803035736, "layer": 12, "max_source_mass": 0.3175881803035736, "mixer_index": 22, "normalized_entropy": 0.976687238475429, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2736307978630066, "median": 0.27999642491340637, "p05": 0.18969643115997314, "p95": 0.3464604914188385 }, { "label": "completed_group_1", "mean": 0.1722733974456787, "median": 0.17083553969860077, "p05": 0.1506597101688385, "p95": 0.1988438367843628 }, { "label": "completed_group_2", "mean": 0.2365075945854187, "median": 0.24049004912376404, "p05": 0.19099700450897217, "p95": 0.26942259073257446 }, { "label": "current_group_partial", "mean": 0.3175881803035736, "median": 0.3092706799507141, "p05": 0.268545538187027, "p95": 0.3924510180950165 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3203014135360718, "group": 3, "latest_source_mass": 0.3395293354988098, "layer": 12, "max_source_mass": 0.3395293354988098, "mixer_index": 23, "normalized_entropy": 0.9523961508935896, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28207293152809143, "median": 0.29242122173309326, "p05": 0.1354818046092987, "p95": 0.4195851981639862 }, { "label": "completed_group_1", "mean": 0.15420743823051453, "median": 0.15396761894226074, "p05": 0.11267110705375671, "p95": 0.1951446533203125 }, { "label": "completed_group_2", "mean": 0.22419027984142303, "median": 0.22542357444763184, "p05": 0.1655150204896927, "p95": 0.2747117578983307 }, { "label": "current_group_partial", "mean": 0.3395293354988098, "median": 0.32681703567504883, "p05": 0.2524653375148773, "p95": 0.46652013063430786 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3734662532806396, "group": 4, "latest_source_mass": 0.283242404460907, "layer": 13, "max_source_mass": 0.283242404460907, "mixer_index": 24, "normalized_entropy": 0.9907464762181619, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2766023874282837, "median": 0.282479852437973, "p05": 0.22528076171875, "p95": 0.3097468614578247 }, { "label": "completed_group_1", "mean": 0.21176087856292725, "median": 0.2058793306350708, "p05": 0.18811854720115662, "p95": 0.2617422640323639 }, { "label": "completed_group_2", "mean": 0.22839435935020447, "median": 0.22951346635818481, "p05": 0.20004916191101074, "p95": 0.2551718056201935 }, { "label": "completed_group_3", "mean": 0.283242404460907, "median": 0.2849352955818176, "p05": 0.2205706238746643, "p95": 0.34444743394851685 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5325813293457031, "group": 4, "latest_source_mass": 0.27591708302497864, "layer": 13, "max_source_mass": 0.27591708302497864, "mixer_index": 25, "normalized_entropy": 0.9522463199762954, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17451074719429016, "median": 0.16826045513153076, "p05": 0.08149556815624237, "p95": 0.3126368522644043 }, { "label": "completed_group_1", "mean": 0.10643941164016724, "median": 0.10269765555858612, "p05": 0.08178127557039261, "p95": 0.13947664201259613 }, { "label": "completed_group_2", "mean": 0.1691833734512329, "median": 0.17118802666664124, "p05": 0.12382284551858902, "p95": 0.2069271057844162 }, { "label": "completed_group_3", "mean": 0.27394935488700867, "median": 0.27278125286102295, "p05": 0.2122008204460144, "p95": 0.33945947885513306 }, { "label": "current_group_partial", "mean": 0.27591708302497864, "median": 0.27714359760284424, "p05": 0.21523593366146088, "p95": 0.33170604705810547 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.560360074043274, "group": 4, "latest_source_mass": 0.3255118429660797, "layer": 14, "max_source_mass": 0.3255118429660797, "mixer_index": 26, "normalized_entropy": 0.9695062244951087, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19377154111862183, "median": 0.19472965598106384, "p05": 0.17005598545074463, "p95": 0.22297024726867676 }, { "label": "completed_group_1", "mean": 0.17211595177650452, "median": 0.17136064171791077, "p05": 0.1543424427509308, "p95": 0.1928609162569046 }, { "label": "completed_group_2", "mean": 0.14605136215686798, "median": 0.14629822969436646, "p05": 0.13246256113052368, "p95": 0.15914371609687805 }, { "label": "completed_group_3", "mean": 0.16254930198192596, "median": 0.16272446513175964, "p05": 0.14075018465518951, "p95": 0.18463709950447083 }, { "label": "current_group_partial", "mean": 0.3255118429660797, "median": 0.3261105418205261, "p05": 0.3035835921764374, "p95": 0.3457673192024231 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.540909767150879, "group": 4, "latest_source_mass": 0.2995486855506897, "layer": 14, "max_source_mass": 0.2995486855506897, "mixer_index": 27, "normalized_entropy": 0.9574210693349581, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21921303868293762, "median": 0.21722173690795898, "p05": 0.1413341462612152, "p95": 0.2906968593597412 }, { "label": "completed_group_1", "mean": 0.10205383598804474, "median": 0.09984464198350906, "p05": 0.07455356419086456, "p95": 0.13362759351730347 }, { "label": "completed_group_2", "mean": 0.16381604969501495, "median": 0.16600549221038818, "p05": 0.11865930259227753, "p95": 0.2019941210746765 }, { "label": "completed_group_3", "mean": 0.215368390083313, "median": 0.2136734575033188, "p05": 0.1728287935256958, "p95": 0.2628125846385956 }, { "label": "current_group_partial", "mean": 0.2995486855506897, "median": 0.2982075810432434, "p05": 0.23863884806632996, "p95": 0.3662579655647278 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5142419338226318, "group": 4, "latest_source_mass": 0.369165301322937, "layer": 15, "max_source_mass": 0.369165301322937, "mixer_index": 28, "normalized_entropy": 0.940851412859105, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2000826597213745, "median": 0.20369741320610046, "p05": 0.15802279114723206, "p95": 0.24072687327861786 }, { "label": "completed_group_1", "mean": 0.15480534732341766, "median": 0.15228059887886047, "p05": 0.13543802499771118, "p95": 0.1815909892320633 }, { "label": "completed_group_2", "mean": 0.10827909409999847, "median": 0.10761409997940063, "p05": 0.09670238941907883, "p95": 0.12229038774967194 }, { "label": "completed_group_3", "mean": 0.16766759753227234, "median": 0.16637766361236572, "p05": 0.1398160457611084, "p95": 0.20042110979557037 }, { "label": "current_group_partial", "mean": 0.369165301322937, "median": 0.3697334825992584, "p05": 0.3279486298561096, "p95": 0.40613043308258057 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5333527326583862, "group": 4, "latest_source_mass": 0.29974091053009033, "layer": 15, "max_source_mass": 0.29974091053009033, "mixer_index": 29, "normalized_entropy": 0.9527256198031004, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21770668029785156, "median": 0.19699469208717346, "p05": 0.1311802864074707, "p95": 0.3720983564853668 }, { "label": "completed_group_1", "mean": 0.10525081306695938, "median": 0.10493603348731995, "p05": 0.08330865204334259, "p95": 0.12839283049106598 }, { "label": "completed_group_2", "mean": 0.15970060229301453, "median": 0.16236470639705658, "p05": 0.11309029906988144, "p95": 0.19861777126789093 }, { "label": "completed_group_3", "mean": 0.217601016163826, "median": 0.21904586255550385, "p05": 0.16268108785152435, "p95": 0.2699928283691406 }, { "label": "current_group_partial", "mean": 0.29974091053009033, "median": 0.30079299211502075, "p05": 0.22156940400600433, "p95": 0.37281978130340576 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.512608289718628, "group": 4, "latest_source_mass": 0.35851576924324036, "layer": 16, "max_source_mass": 0.35851576924324036, "mixer_index": 30, "normalized_entropy": 0.9398363727066501, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21392285823822021, "median": 0.22146853804588318, "p05": 0.1518818885087967, "p95": 0.25874194502830505 }, { "label": "completed_group_1", "mean": 0.1860550194978714, "median": 0.1838144212961197, "p05": 0.1660098135471344, "p95": 0.21311265230178833 }, { "label": "completed_group_2", "mean": 0.10291537642478943, "median": 0.1024642288684845, "p05": 0.09207107126712799, "p95": 0.11504144221544266 }, { "label": "completed_group_3", "mean": 0.1385909616947174, "median": 0.13646918535232544, "p05": 0.11220520734786987, "p95": 0.1728362739086151 }, { "label": "current_group_partial", "mean": 0.35851576924324036, "median": 0.3558155298233032, "p05": 0.31809645891189575, "p95": 0.40622422099113464 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5262645483016968, "group": 4, "latest_source_mass": 0.3142552077770233, "layer": 16, "max_source_mass": 0.3142552077770233, "mixer_index": 31, "normalized_entropy": 0.9483214832396903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20246261358261108, "median": 0.18459102511405945, "p05": 0.12276223301887512, "p95": 0.38892465829849243 }, { "label": "completed_group_1", "mean": 0.10530521720647812, "median": 0.10532709956169128, "p05": 0.0785030871629715, "p95": 0.1342671513557434 }, { "label": "completed_group_2", "mean": 0.16650916635990143, "median": 0.17111444473266602, "p05": 0.1085265576839447, "p95": 0.2094469517469406 }, { "label": "completed_group_3", "mean": 0.21146778762340546, "median": 0.2099919468164444, "p05": 0.15801574289798737, "p95": 0.26882117986679077 }, { "label": "current_group_partial", "mean": 0.3142552077770233, "median": 0.31609731912612915, "p05": 0.22627028822898865, "p95": 0.397154837846756 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5794764757156372, "group": 5, "latest_source_mass": 0.2760102152824402, "layer": 17, "max_source_mass": 0.2760102152824402, "mixer_index": 32, "normalized_entropy": 0.9813839126772218, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2210189402103424, "median": 0.22435861825942993, "p05": 0.17291823029518127, "p95": 0.262204647064209 }, { "label": "completed_group_1", "mean": 0.18839938938617706, "median": 0.18652752041816711, "p05": 0.16997642815113068, "p95": 0.2126670628786087 }, { "label": "completed_group_2", "mean": 0.14016608893871307, "median": 0.14067986607551575, "p05": 0.1265786588191986, "p95": 0.15091627836227417 }, { "label": "completed_group_3", "mean": 0.17440536618232727, "median": 0.17192018032073975, "p05": 0.15062645077705383, "p95": 0.20691406726837158 }, { "label": "completed_group_4", "mean": 0.2760102152824402, "median": 0.27511972188949585, "p05": 0.24412119388580322, "p95": 0.30901142954826355 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6602650880813599, "group": 5, "latest_source_mass": 0.3575916588306427, "layer": 17, "max_source_mass": 0.3575916588306427, "mixer_index": 33, "normalized_entropy": 0.9266115885502495, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12695768475532532, "median": 0.11564654856920242, "p05": 0.07159632444381714, "p95": 0.21949690580368042 }, { "label": "completed_group_1", "mean": 0.08036886155605316, "median": 0.07964898645877838, "p05": 0.06650256365537643, "p95": 0.09638062119483948 }, { "label": "completed_group_2", "mean": 0.12672865390777588, "median": 0.1287270039319992, "p05": 0.09939455986022949, "p95": 0.14712141454219818 }, { "label": "completed_group_3", "mean": 0.15391482412815094, "median": 0.15372109413146973, "p05": 0.13333335518836975, "p95": 0.17581847310066223 }, { "label": "completed_group_4", "mean": 0.154438316822052, "median": 0.15402798354625702, "p05": 0.12481103092432022, "p95": 0.18498417735099792 }, { "label": "current_group_partial", "mean": 0.3575916588306427, "median": 0.36001941561698914, "p05": 0.3110165596008301, "p95": 0.39699214696884155 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.5925545692443848, "group": 5, "latest_source_mass": 0.41908538341522217, "layer": 18, "max_source_mass": 0.41908538341522217, "mixer_index": 34, "normalized_entropy": 0.8888216284580353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14075040817260742, "median": 0.1455841362476349, "p05": 0.09744156897068024, "p95": 0.1818566620349884 }, { "label": "completed_group_1", "mean": 0.11028306186199188, "median": 0.10887216031551361, "p05": 0.09629634767770767, "p95": 0.12874113023281097 }, { "label": "completed_group_2", "mean": 0.07537610828876495, "median": 0.07517752051353455, "p05": 0.0666593387722969, "p95": 0.08491044491529465 }, { "label": "completed_group_3", "mean": 0.10244421660900116, "median": 0.10141433775424957, "p05": 0.08653581142425537, "p95": 0.12175601720809937 }, { "label": "completed_group_4", "mean": 0.1520608365535736, "median": 0.1513972282409668, "p05": 0.1301831305027008, "p95": 0.17589205503463745 }, { "label": "current_group_partial", "mean": 0.41908538341522217, "median": 0.4191858768463135, "p05": 0.3649080991744995, "p95": 0.4721684455871582 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5918848514556885, "group": 5, "latest_source_mass": 0.4151553511619568, "layer": 18, "max_source_mass": 0.4151553511619568, "mixer_index": 35, "normalized_entropy": 0.8884478518433735, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11773698031902313, "median": 0.10857899487018585, "p05": 0.06324320286512375, "p95": 0.22339528799057007 }, { "label": "completed_group_1", "mean": 0.07344591617584229, "median": 0.07228962332010269, "p05": 0.05982384458184242, "p95": 0.09060496836900711 }, { "label": "completed_group_2", "mean": 0.11976176500320435, "median": 0.12151201069355011, "p05": 0.08953659236431122, "p95": 0.14441059529781342 }, { "label": "completed_group_3", "mean": 0.14735516905784607, "median": 0.14620742201805115, "p05": 0.11767779290676117, "p95": 0.17902395129203796 }, { "label": "completed_group_4", "mean": 0.126544788479805, "median": 0.12564796209335327, "p05": 0.09858805686235428, "p95": 0.1567486822605133 }, { "label": "current_group_partial", "mean": 0.4151553511619568, "median": 0.4184337854385376, "p05": 0.3375611901283264, "p95": 0.4846380352973938 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6208878755569458, "group": 5, "latest_source_mass": 0.3991435170173645, "layer": 19, "max_source_mass": 0.3991435170173645, "mixer_index": 36, "normalized_entropy": 0.9046347477964072, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14785674214363098, "median": 0.15170978009700775, "p05": 0.09146977216005325, "p95": 0.20172694325447083 }, { "label": "completed_group_1", "mean": 0.12492769956588745, "median": 0.12345598638057709, "p05": 0.10790181905031204, "p95": 0.14623555541038513 }, { "label": "completed_group_2", "mean": 0.08781818300485611, "median": 0.08839264512062073, "p05": 0.07453933358192444, "p95": 0.09894395619630814 }, { "label": "completed_group_3", "mean": 0.10019385814666748, "median": 0.09901996701955795, "p05": 0.08430047333240509, "p95": 0.12013883143663406 }, { "label": "completed_group_4", "mean": 0.14006000757217407, "median": 0.13918349146842957, "p05": 0.11816112697124481, "p95": 0.16475799679756165 }, { "label": "current_group_partial", "mean": 0.3991435170173645, "median": 0.39726996421813965, "p05": 0.34116071462631226, "p95": 0.4634247422218323 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5470075607299805, "group": 5, "latest_source_mass": 0.44330859184265137, "layer": 19, "max_source_mass": 0.44330859184265137, "mixer_index": 37, "normalized_entropy": 0.8634013589985261, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12043030560016632, "median": 0.1012125313282013, "p05": 0.061379045248031616, "p95": 0.25343188643455505 }, { "label": "completed_group_1", "mean": 0.06855648756027222, "median": 0.06761305779218674, "p05": 0.05337192490696907, "p95": 0.08790726959705353 }, { "label": "completed_group_2", "mean": 0.11753211170434952, "median": 0.1186976358294487, "p05": 0.08412563800811768, "p95": 0.14895018935203552 }, { "label": "completed_group_3", "mean": 0.1339646875858307, "median": 0.13253431022167206, "p05": 0.09857159107923508, "p95": 0.1730121374130249 }, { "label": "completed_group_4", "mean": 0.1162077933549881, "median": 0.11592038720846176, "p05": 0.08609852194786072, "p95": 0.14767511188983917 }, { "label": "current_group_partial", "mean": 0.44330859184265137, "median": 0.4434061646461487, "p05": 0.3389148414134979, "p95": 0.5490255355834961 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.683492660522461, "group": 5, "latest_source_mass": 0.3365415334701538, "layer": 20, "max_source_mass": 0.3365415334701538, "mixer_index": 38, "normalized_entropy": 0.9395751435586169, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1658952683210373, "median": 0.1726911962032318, "p05": 0.10183650255203247, "p95": 0.22522984445095062 }, { "label": "completed_group_1", "mean": 0.1435280442237854, "median": 0.14210543036460876, "p05": 0.12450659275054932, "p95": 0.16782894730567932 }, { "label": "completed_group_2", "mean": 0.09461991488933563, "median": 0.0948714017868042, "p05": 0.08168491721153259, "p95": 0.10723742842674255 }, { "label": "completed_group_3", "mean": 0.10784181207418442, "median": 0.10693192481994629, "p05": 0.08930590003728867, "p95": 0.12958656251430511 }, { "label": "completed_group_4", "mean": 0.15157344937324524, "median": 0.15053030848503113, "p05": 0.128616601228714, "p95": 0.17799487709999084 }, { "label": "current_group_partial", "mean": 0.3365415334701538, "median": 0.33349889516830444, "p05": 0.2778944969177246, "p95": 0.40724658966064453 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5540567636489868, "group": 5, "latest_source_mass": 0.42266085743904114, "layer": 20, "max_source_mass": 0.42266085743904114, "mixer_index": 39, "normalized_entropy": 0.8673355940563396, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12987135350704193, "median": 0.1083056628704071, "p05": 0.03783264383673668, "p95": 0.3301126956939697 }, { "label": "completed_group_1", "mean": 0.07178181409835815, "median": 0.07081792503595352, "p05": 0.05334944650530815, "p95": 0.09311570972204208 }, { "label": "completed_group_2", "mean": 0.12132491171360016, "median": 0.12282539904117584, "p05": 0.07926258444786072, "p95": 0.15763431787490845 }, { "label": "completed_group_3", "mean": 0.1441602110862732, "median": 0.14273977279663086, "p05": 0.09847753494977951, "p95": 0.19304539263248444 }, { "label": "completed_group_4", "mean": 0.11020088195800781, "median": 0.10934801399707794, "p05": 0.07139573246240616, "p95": 0.1517774760723114 }, { "label": "current_group_partial", "mean": 0.42266085743904114, "median": 0.42266106605529785, "p05": 0.28481048345565796, "p95": 0.5570361018180847 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7208337783813477, "group": 6, "latest_source_mass": 0.27070939540863037, "layer": 21, "max_source_mass": 0.27070939540863037, "mixer_index": 40, "normalized_entropy": 0.9604156182429642, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.18048501014709473, "median": 0.18919861316680908, "p05": 0.11199048161506653, "p95": 0.23807701468467712 }, { "label": "completed_group_1", "mean": 0.1685236394405365, "median": 0.1669977903366089, "p05": 0.14678362011909485, "p95": 0.19381441175937653 }, { "label": "completed_group_2", "mean": 0.08726631104946136, "median": 0.08729851245880127, "p05": 0.07629868388175964, "p95": 0.0980045348405838 }, { "label": "completed_group_3", "mean": 0.11160479485988617, "median": 0.1099197044968605, "p05": 0.09190923720598221, "p95": 0.13723208010196686 }, { "label": "completed_group_4", "mean": 0.18141084909439087, "median": 0.18020854890346527, "p05": 0.15656134486198425, "p95": 0.2108723670244217 }, { "label": "completed_group_5", "mean": 0.27070939540863037, "median": 0.2684924602508545, "p05": 0.2159801721572876, "p95": 0.33192309737205505 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6729483604431152, "group": 6, "latest_source_mass": 0.3803577423095703, "layer": 21, "max_source_mass": 0.3803577423095703, "mixer_index": 41, "normalized_entropy": 0.8597253893019091, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09229261428117752, "median": 0.07798201590776443, "p05": 0.04784023016691208, "p95": 0.20285779237747192 }, { "label": "completed_group_1", "mean": 0.051420148462057114, "median": 0.05041736364364624, "p05": 0.040625106543302536, "p95": 0.06547587364912033 }, { "label": "completed_group_2", "mean": 0.08079522103071213, "median": 0.08136115223169327, "p05": 0.05947277322411537, "p95": 0.10017222166061401 }, { "label": "completed_group_3", "mean": 0.09550285339355469, "median": 0.09445887804031372, "p05": 0.07334083318710327, "p95": 0.12144046276807785 }, { "label": "completed_group_4", "mean": 0.07590493559837341, "median": 0.07492390275001526, "p05": 0.055510398000478745, "p95": 0.10084333270788193 }, { "label": "completed_group_5", "mean": 0.22372648119926453, "median": 0.22069036960601807, "p05": 0.1407623589038849, "p95": 0.32290777564048767 }, { "label": "current_group_partial", "mean": 0.3803577423095703, "median": 0.37903881072998047, "p05": 0.2855539321899414, "p95": 0.476589173078537 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.710235595703125, "group": 6, "latest_source_mass": 0.4011716842651367, "layer": 22, "max_source_mass": 0.4011716842651367, "mixer_index": 42, "normalized_entropy": 0.8788872376935791, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11657392978668213, "median": 0.11877645552158356, "p05": 0.06819256395101547, "p95": 0.1684979349374771 }, { "label": "completed_group_1", "mean": 0.10617930442094803, "median": 0.10509790480136871, "p05": 0.090352863073349, "p95": 0.12587188184261322 }, { "label": "completed_group_2", "mean": 0.061112720519304276, "median": 0.061012350022792816, "p05": 0.05164862796664238, "p95": 0.07133162766695023 }, { "label": "completed_group_3", "mean": 0.06796726584434509, "median": 0.06712733209133148, "p05": 0.05444999784231186, "p95": 0.08481802046298981 }, { "label": "completed_group_4", "mean": 0.103337861597538, "median": 0.10192783921957016, "p05": 0.0832459107041359, "p95": 0.12805812060832977 }, { "label": "completed_group_5", "mean": 0.14365722239017487, "median": 0.14159655570983887, "p05": 0.10498540848493576, "p95": 0.19020715355873108 }, { "label": "current_group_partial", "mean": 0.4011716842651367, "median": 0.40084803104400635, "p05": 0.32083970308303833, "p95": 0.47994357347488403 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5640671253204346, "group": 6, "latest_source_mass": 0.4689176678657532, "layer": 22, "max_source_mass": 0.4689176678657532, "mixer_index": 43, "normalized_entropy": 0.8037715030571925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.090422622859478, "median": 0.0774901732802391, "p05": 0.04492149129509926, "p95": 0.1988787055015564 }, { "label": "completed_group_1", "mean": 0.04561382159590721, "median": 0.044634923338890076, "p05": 0.03501313924789429, "p95": 0.05900419130921364 }, { "label": "completed_group_2", "mean": 0.07565715163946152, "median": 0.07550699263811111, "p05": 0.053458306938409805, "p95": 0.09821777045726776 }, { "label": "completed_group_3", "mean": 0.08878682553768158, "median": 0.08692863583564758, "p05": 0.06425619125366211, "p95": 0.11807316541671753 }, { "label": "completed_group_4", "mean": 0.06555216014385223, "median": 0.06414324045181274, "p05": 0.04460844025015831, "p95": 0.09090004861354828 }, { "label": "completed_group_5", "mean": 0.165049746632576, "median": 0.15503892302513123, "p05": 0.08368627727031708, "p95": 0.2792147397994995 }, { "label": "current_group_partial", "mean": 0.4689176678657532, "median": 0.46879667043685913, "p05": 0.3151778280735016, "p95": 0.6170428991317749 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7556228637695312, "group": 6, "latest_source_mass": 0.3706088662147522, "layer": 23, "max_source_mass": 0.3706088662147522, "mixer_index": 44, "normalized_entropy": 0.9022116795175967, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13281464576721191, "median": 0.13658174872398376, "p05": 0.08060654997825623, "p95": 0.17402096092700958 }, { "label": "completed_group_1", "mean": 0.12175846099853516, "median": 0.12066285312175751, "p05": 0.10334118455648422, "p95": 0.14384035766124725 }, { "label": "completed_group_2", "mean": 0.07235272228717804, "median": 0.07218228280544281, "p05": 0.059715576469898224, "p95": 0.08584222197532654 }, { "label": "completed_group_3", "mean": 0.0707377940416336, "median": 0.06942892074584961, "p05": 0.05500924959778786, "p95": 0.09052760899066925 }, { "label": "completed_group_4", "mean": 0.10551950335502625, "median": 0.103996641933918, "p05": 0.08408617228269577, "p95": 0.13205930590629578 }, { "label": "completed_group_5", "mean": 0.12620802223682404, "median": 0.12379512935876846, "p05": 0.08937962353229523, "p95": 0.17139491438865662 }, { "label": "current_group_partial", "mean": 0.3706088662147522, "median": 0.3705657124519348, "p05": 0.282781720161438, "p95": 0.4561466872692108 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.594892978668213, "group": 6, "latest_source_mass": 0.4636557698249817, "layer": 23, "max_source_mass": 0.4636557698249817, "mixer_index": 45, "normalized_entropy": 0.8196128579947488, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10721131414175034, "median": 0.10240983963012695, "p05": 0.05563424155116081, "p95": 0.18401460349559784 }, { "label": "completed_group_1", "mean": 0.053925663232803345, "median": 0.0527028888463974, "p05": 0.04038037359714508, "p95": 0.07054222375154495 }, { "label": "completed_group_2", "mean": 0.07673212885856628, "median": 0.07705660909414291, "p05": 0.05211106687784195, "p95": 0.10031480342149734 }, { "label": "completed_group_3", "mean": 0.08865120261907578, "median": 0.08699032664299011, "p05": 0.06242028623819351, "p95": 0.11899784207344055 }, { "label": "completed_group_4", "mean": 0.06888366490602493, "median": 0.06751501560211182, "p05": 0.04784276708960533, "p95": 0.09609999507665634 }, { "label": "completed_group_5", "mean": 0.14094024896621704, "median": 0.13063763082027435, "p05": 0.07297889143228531, "p95": 0.24012833833694458 }, { "label": "current_group_partial", "mean": 0.4636557698249817, "median": 0.46312135457992554, "p05": 0.31259939074516296, "p95": 0.6109440326690674 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7775511741638184, "group": 6, "latest_source_mass": 0.3569435477256775, "layer": 24, "max_source_mass": 0.3569435477256775, "mixer_index": 46, "normalized_entropy": 0.9134806018801903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13637229800224304, "median": 0.13456633687019348, "p05": 0.09586701542139053, "p95": 0.18876218795776367 }, { "label": "completed_group_1", "mean": 0.10478455573320389, "median": 0.10326151549816132, "p05": 0.0880400538444519, "p95": 0.12566551566123962 }, { "label": "completed_group_2", "mean": 0.07710614800453186, "median": 0.07678486406803131, "p05": 0.06292064487934113, "p95": 0.09218747913837433 }, { "label": "completed_group_3", "mean": 0.08548048138618469, "median": 0.08430111408233643, "p05": 0.0665682703256607, "p95": 0.10857507586479187 }, { "label": "completed_group_4", "mean": 0.11000140011310577, "median": 0.10904969274997711, "p05": 0.0881001353263855, "p95": 0.13561299443244934 }, { "label": "completed_group_5", "mean": 0.12931162118911743, "median": 0.12692990899085999, "p05": 0.08648453652858734, "p95": 0.1814313679933548 }, { "label": "current_group_partial", "mean": 0.3569435477256775, "median": 0.3568228483200073, "p05": 0.26841166615486145, "p95": 0.4493965804576874 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.654036045074463, "group": 6, "latest_source_mass": 0.4265251159667969, "layer": 24, "max_source_mass": 0.4265251159667969, "mixer_index": 47, "normalized_entropy": 0.8500063817835848, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10232305526733398, "median": 0.08623291552066803, "p05": 0.04679499566555023, "p95": 0.23077142238616943 }, { "label": "completed_group_1", "mean": 0.061968062072992325, "median": 0.060845747590065, "p05": 0.0479469932615757, "p95": 0.07941868156194687 }, { "label": "completed_group_2", "mean": 0.08900615572929382, "median": 0.08864577114582062, "p05": 0.0641680359840393, "p95": 0.11413820832967758 }, { "label": "completed_group_3", "mean": 0.1005433052778244, "median": 0.09813467413187027, "p05": 0.0731433779001236, "p95": 0.1343638002872467 }, { "label": "completed_group_4", "mean": 0.07864196598529816, "median": 0.07639941573143005, "p05": 0.05487418174743652, "p95": 0.10941869765520096 }, { "label": "completed_group_5", "mean": 0.14099235832691193, "median": 0.12885531783103943, "p05": 0.07315157353878021, "p95": 0.24481485784053802 }, { "label": "current_group_partial", "mean": 0.4265251159667969, "median": 0.42903727293014526, "p05": 0.2803359627723694, "p95": 0.5635722875595093 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.84956693649292, "group": 7, "latest_source_mass": 0.28775152564048767, "layer": 25, "max_source_mass": 0.28775152564048767, "mixer_index": 48, "normalized_entropy": 0.9504893827656096, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14851760864257812, "median": 0.14813844859600067, "p05": 0.10312555730342865, "p95": 0.20065288245677948 }, { "label": "completed_group_1", "mean": 0.11630287766456604, "median": 0.11500488221645355, "p05": 0.09825178980827332, "p95": 0.13750022649765015 }, { "label": "completed_group_2", "mean": 0.08512166142463684, "median": 0.08458003401756287, "p05": 0.0710662305355072, "p95": 0.10092862695455551 }, { "label": "completed_group_3", "mean": 0.09215595573186874, "median": 0.09079311043024063, "p05": 0.07304700464010239, "p95": 0.11515989899635315 }, { "label": "completed_group_4", "mean": 0.1263950914144516, "median": 0.12563186883926392, "p05": 0.10441911220550537, "p95": 0.1519687920808792 }, { "label": "completed_group_5", "mean": 0.14375528693199158, "median": 0.1422564834356308, "p05": 0.10313796997070312, "p95": 0.1929876059293747 }, { "label": "completed_group_6", "mean": 0.28775152564048767, "median": 0.28676605224609375, "p05": 0.21690022945404053, "p95": 0.3624127507209778 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7161309719085693, "group": 7, "latest_source_mass": 0.32536065578460693, "layer": 25, "max_source_mass": 0.32536065578460693, "mixer_index": 49, "normalized_entropy": 0.8252845475628168, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0837959572672844, "median": 0.0849243700504303, "p05": 0.0271806251257658, "p95": 0.15180703997612 }, { "label": "completed_group_1", "mean": 0.04434927552938461, "median": 0.04302988201379776, "p05": 0.03217232972383499, "p95": 0.06002792343497276 }, { "label": "completed_group_2", "mean": 0.056708935648202896, "median": 0.05563671886920929, "p05": 0.03992946073412895, "p95": 0.07731831818819046 }, { "label": "completed_group_3", "mean": 0.06495873630046844, "median": 0.06223570555448532, "p05": 0.04220382124185562, "p95": 0.0949486792087555 }, { "label": "completed_group_4", "mean": 0.04885463789105415, "median": 0.046898528933525085, "p05": 0.030923232436180115, "p95": 0.0743112787604332 }, { "label": "completed_group_5", "mean": 0.09997115284204483, "median": 0.08650931715965271, "p05": 0.04111029952764511, "p95": 0.20292627811431885 }, { "label": "completed_group_6", "mean": 0.2760006785392761, "median": 0.27142390608787537, "p05": 0.13761089742183685, "p95": 0.42605364322662354 }, { "label": "current_group_partial", "mean": 0.32536065578460693, "median": 0.3348004221916199, "p05": 0.10654139518737793, "p95": 0.5236068964004517 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9285151958465576, "group": 7, "latest_source_mass": 0.2804902195930481, "layer": 26, "max_source_mass": 0.2804902195930481, "mixer_index": 50, "normalized_entropy": 0.9274197697756124, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10564712435007095, "median": 0.10497278720140457, "p05": 0.04608207941055298, "p95": 0.16350959241390228 }, { "label": "completed_group_1", "mean": 0.09173665940761566, "median": 0.09041133522987366, "p05": 0.07509034872055054, "p95": 0.11238400638103485 }, { "label": "completed_group_2", "mean": 0.07135222852230072, "median": 0.06981296837329865, "p05": 0.0522611029446125, "p95": 0.09555281698703766 }, { "label": "completed_group_3", "mean": 0.07045470923185349, "median": 0.06927657127380371, "p05": 0.055566973984241486, "p95": 0.08908769488334656 }, { "label": "completed_group_4", "mean": 0.09785081446170807, "median": 0.0971219539642334, "p05": 0.07590530812740326, "p95": 0.12285276502370834 }, { "label": "completed_group_5", "mean": 0.10425601899623871, "median": 0.10151226818561554, "p05": 0.06786561012268066, "p95": 0.15156632661819458 }, { "label": "completed_group_6", "mean": 0.1782122254371643, "median": 0.1762663722038269, "p05": 0.11771149188280106, "p95": 0.24932436645030975 }, { "label": "current_group_partial", "mean": 0.2804902195930481, "median": 0.2783043086528778, "p05": 0.14811429381370544, "p95": 0.4168073534965515 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7219383716583252, "group": 7, "latest_source_mass": 0.3661728501319885, "layer": 26, "max_source_mass": 0.3661728501319885, "mixer_index": 51, "normalized_entropy": 0.8280773165026276, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0923302173614502, "median": 0.08522969484329224, "p05": 0.056062981486320496, "p95": 0.1509525179862976 }, { "label": "completed_group_1", "mean": 0.04646994173526764, "median": 0.04543843865394592, "p05": 0.032575178891420364, "p95": 0.06306323409080505 }, { "label": "completed_group_2", "mean": 0.05750556290149689, "median": 0.05649096518754959, "p05": 0.03895717114210129, "p95": 0.07940060645341873 }, { "label": "completed_group_3", "mean": 0.06549686938524246, "median": 0.062135741114616394, "p05": 0.04169235751032829, "p95": 0.09948549419641495 }, { "label": "completed_group_4", "mean": 0.054404959082603455, "median": 0.05241036415100098, "p05": 0.03378590941429138, "p95": 0.08157843351364136 }, { "label": "completed_group_5", "mean": 0.08427833020687103, "median": 0.07576382160186768, "p05": 0.041103728115558624, "p95": 0.15204203128814697 }, { "label": "completed_group_6", "mean": 0.2333412617444992, "median": 0.22215056419372559, "p05": 0.10060560703277588, "p95": 0.3983197808265686 }, { "label": "current_group_partial", "mean": 0.3661728501319885, "median": 0.3673480749130249, "p05": 0.16410952806472778, "p95": 0.5635213255882263 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9470824003219604, "group": 7, "latest_source_mass": 0.26050227880477905, "layer": 27, "max_source_mass": 0.26050227880477905, "mixer_index": 52, "normalized_entropy": 0.9363487077155573, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10804469138383865, "median": 0.10379163920879364, "p05": 0.05421917140483856, "p95": 0.17028658092021942 }, { "label": "completed_group_1", "mean": 0.07850409299135208, "median": 0.0776081532239914, "p05": 0.063654825091362, "p95": 0.09574449062347412 }, { "label": "completed_group_2", "mean": 0.07298290729522705, "median": 0.07200999557971954, "p05": 0.05533483251929283, "p95": 0.09299935400485992 }, { "label": "completed_group_3", "mean": 0.08511961251497269, "median": 0.08372659981250763, "p05": 0.06449062377214432, "p95": 0.11086944490671158 }, { "label": "completed_group_4", "mean": 0.09968686103820801, "median": 0.09877584874629974, "p05": 0.07606595754623413, "p95": 0.1266063004732132 }, { "label": "completed_group_5", "mean": 0.11502514779567719, "median": 0.11171717941761017, "p05": 0.07302433997392654, "p95": 0.1681303083896637 }, { "label": "completed_group_6", "mean": 0.18013443052768707, "median": 0.1765173077583313, "p05": 0.11514104157686234, "p95": 0.2599799633026123 }, { "label": "current_group_partial", "mean": 0.26050227880477905, "median": 0.2575443983078003, "p05": 0.13687226176261902, "p95": 0.39756521582603455 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7855088710784912, "group": 7, "latest_source_mass": 0.28804832696914673, "layer": 27, "max_source_mass": 0.28804832696914673, "mixer_index": 53, "normalized_entropy": 0.858648264589397, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08493785560131073, "median": 0.05951457470655441, "p05": 0.02441210113465786, "p95": 0.22703105211257935 }, { "label": "completed_group_1", "mean": 0.05972691997885704, "median": 0.058127306401729584, "p05": 0.03591413050889969, "p95": 0.08903861790895462 }, { "label": "completed_group_2", "mean": 0.07017223536968231, "median": 0.06906847655773163, "p05": 0.04151676222681999, "p95": 0.10273467004299164 }, { "label": "completed_group_3", "mean": 0.07963316142559052, "median": 0.07547932863235474, "p05": 0.042773112654685974, "p95": 0.12874338030815125 }, { "label": "completed_group_4", "mean": 0.0645163506269455, "median": 0.061876002699136734, "p05": 0.0366181954741478, "p95": 0.10098200291395187 }, { "label": "completed_group_5", "mean": 0.11586236953735352, "median": 0.10182450711727142, "p05": 0.048887670040130615, "p95": 0.2313569337129593 }, { "label": "completed_group_6", "mean": 0.23710274696350098, "median": 0.22610372304916382, "p05": 0.10011275857686996, "p95": 0.4084784686565399 }, { "label": "current_group_partial", "mean": 0.28804832696914673, "median": 0.27438053488731384, "p05": 0.07688213139772415, "p95": 0.5529112815856934 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.973182201385498, "group": 7, "latest_source_mass": 0.23531918227672577, "layer": 28, "max_source_mass": 0.23531918227672577, "mixer_index": 54, "normalized_entropy": 0.9489000589030754, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.104981929063797, "median": 0.09386023879051208, "p05": 0.04212673753499985, "p95": 0.18614700436592102 }, { "label": "completed_group_1", "mean": 0.0840599462389946, "median": 0.08250434696674347, "p05": 0.05170809105038643, "p95": 0.12236068397760391 }, { "label": "completed_group_2", "mean": 0.09310521930456161, "median": 0.0931512713432312, "p05": 0.06374019384384155, "p95": 0.12329407781362534 }, { "label": "completed_group_3", "mean": 0.11566174775362015, "median": 0.11439734697341919, "p05": 0.07886623591184616, "p95": 0.1571493148803711 }, { "label": "completed_group_4", "mean": 0.0963074117898941, "median": 0.09477876126766205, "p05": 0.06686149537563324, "p95": 0.13022682070732117 }, { "label": "completed_group_5", "mean": 0.1096896305680275, "median": 0.10702179372310638, "p05": 0.06579374521970749, "p95": 0.1643676459789276 }, { "label": "completed_group_6", "mean": 0.16087493300437927, "median": 0.15455034375190735, "p05": 0.09876372665166855, "p95": 0.24324838817119598 }, { "label": "current_group_partial", "mean": 0.23531918227672577, "median": 0.22382870316505432, "p05": 0.11656329780817032, "p95": 0.38286590576171875 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.802741289138794, "group": 7, "latest_source_mass": 0.2497689425945282, "layer": 28, "max_source_mass": 0.2913346588611603, "mixer_index": 55, "normalized_entropy": 0.8669353059487718, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08190082013607025, "median": 0.06415966153144836, "p05": 0.039168842136859894, "p95": 0.23471535742282867 }, { "label": "completed_group_1", "mean": 0.05632811039686203, "median": 0.055233441293239594, "p05": 0.03936203569173813, "p95": 0.076178178191185 }, { "label": "completed_group_2", "mean": 0.07102417945861816, "median": 0.06932097673416138, "p05": 0.04159427434206009, "p95": 0.1046731173992157 }, { "label": "completed_group_3", "mean": 0.08421063423156738, "median": 0.0812690332531929, "p05": 0.0486646331846714, "p95": 0.12889066338539124 }, { "label": "completed_group_4", "mean": 0.06341387331485748, "median": 0.061601825058460236, "p05": 0.03807957097887993, "p95": 0.09577237069606781 }, { "label": "completed_group_5", "mean": 0.10201875865459442, "median": 0.09330011904239655, "p05": 0.05116701498627663, "p95": 0.18108239769935608 }, { "label": "completed_group_6", "mean": 0.2913346588611603, "median": 0.28823426365852356, "p05": 0.13295966386795044, "p95": 0.4586564898490906 }, { "label": "current_group_partial", "mean": 0.2497689425945282, "median": 0.234481081366539, "p05": 0.09318265318870544, "p95": 0.4530518054962158 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.945815086364746, "group": 8, "latest_source_mass": 0.153213769197464, "layer": 29, "max_source_mass": 0.2451319396495819, "mixer_index": 56, "normalized_entropy": 0.9357392585284499, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1064736545085907, "median": 0.0881282389163971, "p05": 0.029344500973820686, "p95": 0.23970629274845123 }, { "label": "completed_group_1", "mean": 0.07925166189670563, "median": 0.07691921293735504, "p05": 0.062442928552627563, "p95": 0.1030145138502121 }, { "label": "completed_group_2", "mean": 0.07823626697063446, "median": 0.07715678215026855, "p05": 0.059018202126026154, "p95": 0.10055352747440338 }, { "label": "completed_group_3", "mean": 0.09447507560253143, "median": 0.09318602085113525, "p05": 0.0684189647436142, "p95": 0.12381350249052048 }, { "label": "completed_group_4", "mean": 0.09724453091621399, "median": 0.09628183394670486, "p05": 0.0699525699019432, "p95": 0.12776269018650055 }, { "label": "completed_group_5", "mean": 0.1459731012582779, "median": 0.14035329222679138, "p05": 0.08667999505996704, "p95": 0.2258891463279724 }, { "label": "completed_group_6", "mean": 0.2451319396495819, "median": 0.24505369365215302, "p05": 0.16207969188690186, "p95": 0.3307104706764221 }, { "label": "completed_group_7", "mean": 0.153213769197464, "median": 0.13417893648147583, "p05": 0.05588865652680397, "p95": 0.30438560247421265 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8717057704925537, "group": 8, "latest_source_mass": 0.3048541843891144, "layer": 29, "max_source_mass": 0.3048541843891144, "mixer_index": 57, "normalized_entropy": 0.8518500065030654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07336360216140747, "median": 0.0611552819609642, "p05": 0.024239415302872658, "p95": 0.16916418075561523 }, { "label": "completed_group_1", "mean": 0.05720400810241699, "median": 0.05402737483382225, "p05": 0.0314149409532547, "p95": 0.09488057345151901 }, { "label": "completed_group_2", "mean": 0.057025715708732605, "median": 0.05547948181629181, "p05": 0.03480810672044754, "p95": 0.08415934443473816 }, { "label": "completed_group_3", "mean": 0.06655597686767578, "median": 0.06265220046043396, "p05": 0.035200610756874084, "p95": 0.10949824750423431 }, { "label": "completed_group_4", "mean": 0.05032068490982056, "median": 0.047904569655656815, "p05": 0.02614513412117958, "p95": 0.08244554698467255 }, { "label": "completed_group_5", "mean": 0.08329877257347107, "median": 0.0731179416179657, "p05": 0.02939312346279621, "p95": 0.17248544096946716 }, { "label": "completed_group_6", "mean": 0.1901058554649353, "median": 0.18234091997146606, "p05": 0.08719424903392792, "p95": 0.317789226770401 }, { "label": "completed_group_7", "mean": 0.11727118492126465, "median": 0.09272899478673935, "p05": 0.031939513981342316, "p95": 0.2891285717487335 }, { "label": "current_group_partial", "mean": 0.3048541843891144, "median": 0.29432767629623413, "p05": 0.07913826406002045, "p95": 0.5632759928703308 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.09096622467041, "group": 8, "latest_source_mass": 0.13795366883277893, "layer": 30, "max_source_mass": 0.17320571839809418, "mixer_index": 58, "normalized_entropy": 0.951639739623416, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10326676070690155, "median": 0.09009090811014175, "p05": 0.01757553033530712, "p95": 0.2674753665924072 }, { "label": "completed_group_1", "mean": 0.08232800662517548, "median": 0.08093428611755371, "p05": 0.06430500745773315, "p95": 0.10457758605480194 }, { "label": "completed_group_2", "mean": 0.07164021581411362, "median": 0.0711187869310379, "p05": 0.05190654471516609, "p95": 0.09468771517276764 }, { "label": "completed_group_3", "mean": 0.08822821825742722, "median": 0.08667590469121933, "p05": 0.06148773431777954, "p95": 0.12025904655456543 }, { "label": "completed_group_4", "mean": 0.10803373157978058, "median": 0.10627314448356628, "p05": 0.07042569667100906, "p95": 0.1514367014169693 }, { "label": "completed_group_5", "mean": 0.13578416407108307, "median": 0.12929600477218628, "p05": 0.08227657526731491, "p95": 0.21117588877677917 }, { "label": "completed_group_6", "mean": 0.17320571839809418, "median": 0.1694163680076599, "p05": 0.11322733014822006, "p95": 0.24482715129852295 }, { "label": "completed_group_7", "mean": 0.09955951571464539, "median": 0.08973086625337601, "p05": 0.037010837346315384, "p95": 0.1912725567817688 }, { "label": "current_group_partial", "mean": 0.13795366883277893, "median": 0.1256815791130066, "p05": 0.053015124052762985, "p95": 0.2601332366466522 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.96881103515625, "group": 8, "latest_source_mass": 0.21570813655853271, "layer": 30, "max_source_mass": 0.21570813655853271, "mixer_index": 59, "normalized_entropy": 0.896044517007504, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08843819797039032, "median": 0.08496083319187164, "p05": 0.03690534085035324, "p95": 0.1522325873374939 }, { "label": "completed_group_1", "mean": 0.06052382290363312, "median": 0.058369796723127365, "p05": 0.03549504280090332, "p95": 0.09154202789068222 }, { "label": "completed_group_2", "mean": 0.0642135739326477, "median": 0.06355445086956024, "p05": 0.03948400169610977, "p95": 0.09175601601600647 }, { "label": "completed_group_3", "mean": 0.07744920998811722, "median": 0.07566337287425995, "p05": 0.04564609378576279, "p95": 0.11733324825763702 }, { "label": "completed_group_4", "mean": 0.051616813987493515, "median": 0.049985647201538086, "p05": 0.03238597884774208, "p95": 0.07699409872293472 }, { "label": "completed_group_5", "mean": 0.06631231307983398, "median": 0.061356961727142334, "p05": 0.030705859884619713, "p95": 0.1188126802444458 }, { "label": "completed_group_6", "mean": 0.16417551040649414, "median": 0.15454982221126556, "p05": 0.07227650284767151, "p95": 0.28998225927352905 }, { "label": "completed_group_7", "mean": 0.2115624099969864, "median": 0.19997704029083252, "p05": 0.09131578356027603, "p95": 0.3709705173969269 }, { "label": "current_group_partial", "mean": 0.21570813655853271, "median": 0.2005728930234909, "p05": 0.06839637458324432, "p95": 0.4193759858608246 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.124063014984131, "group": 8, "latest_source_mass": 0.1258423924446106, "layer": 31, "max_source_mass": 0.17017140984535217, "mixer_index": 60, "normalized_entropy": 0.9667027380329118, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09220977127552032, "median": 0.09152225404977798, "p05": 0.03251190483570099, "p95": 0.17071731388568878 }, { "label": "completed_group_1", "mean": 0.06839152425527573, "median": 0.06702418625354767, "p05": 0.04446800425648689, "p95": 0.0977444052696228 }, { "label": "completed_group_2", "mean": 0.10039852559566498, "median": 0.09736946225166321, "p05": 0.06476961821317673, "p95": 0.14545989036560059 }, { "label": "completed_group_3", "mean": 0.10902463644742966, "median": 0.10749617218971252, "p05": 0.07769118249416351, "p95": 0.14509552717208862 }, { "label": "completed_group_4", "mean": 0.0849665105342865, "median": 0.08355648815631866, "p05": 0.059977494180202484, "p95": 0.11480176448822021 }, { "label": "completed_group_5", "mean": 0.12137698382139206, "median": 0.11852307617664337, "p05": 0.07902064919471741, "p95": 0.17207489907741547 }, { "label": "completed_group_6", "mean": 0.17017140984535217, "median": 0.16817378997802734, "p05": 0.11521546542644501, "p95": 0.23236754536628723 }, { "label": "completed_group_7", "mean": 0.12761825323104858, "median": 0.1237221211194992, "p05": 0.062459878623485565, "p95": 0.2092883139848709 }, { "label": "current_group_partial", "mean": 0.1258423924446106, "median": 0.11723431199789047, "p05": 0.06504593789577484, "p95": 0.21421310305595398 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9633058309555054, "group": 8, "latest_source_mass": 0.15763342380523682, "layer": 31, "max_source_mass": 0.23511825501918793, "mixer_index": 61, "normalized_entropy": 0.8935389906004497, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.0846441388130188, "median": 0.06999644637107849, "p05": 0.04030589759349823, "p95": 0.20743247866630554 }, { "label": "completed_group_1", "mean": 0.05872901901602745, "median": 0.05653517693281174, "p05": 0.03710630163550377, "p95": 0.08772030472755432 }, { "label": "completed_group_2", "mean": 0.07210008800029755, "median": 0.07183568179607391, "p05": 0.03840115666389465, "p95": 0.10702595114707947 }, { "label": "completed_group_3", "mean": 0.07792744785547256, "median": 0.0762966051697731, "p05": 0.04050692170858383, "p95": 0.122304767370224 }, { "label": "completed_group_4", "mean": 0.059186771512031555, "median": 0.057271093130111694, "p05": 0.03537197411060333, "p95": 0.08958204090595245 }, { "label": "completed_group_5", "mean": 0.07445863634347916, "median": 0.06928195059299469, "p05": 0.03154807910323143, "p95": 0.1351260244846344 }, { "label": "completed_group_6", "mean": 0.23511825501918793, "median": 0.22703276574611664, "p05": 0.0953572615981102, "p95": 0.3982555568218231 }, { "label": "completed_group_7", "mean": 0.18020221590995789, "median": 0.16846294701099396, "p05": 0.07464539259672165, "p95": 0.324230432510376 }, { "label": "current_group_partial", "mean": 0.15763342380523682, "median": 0.12027859687805176, "p05": 0.036883872002363205, "p95": 0.4093724191188812 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1238560676574707, "group": 8, "latest_source_mass": 0.11516246944665909, "layer": 32, "max_source_mass": 0.1766587197780609, "mixer_index": 62, "normalized_entropy": 0.966608552245626, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10231245309114456, "median": 0.10458895564079285, "p05": 0.025718798860907555, "p95": 0.1875312179327011 }, { "label": "completed_group_1", "mean": 0.07575537264347076, "median": 0.07447351515293121, "p05": 0.05878028646111488, "p95": 0.09651422500610352 }, { "label": "completed_group_2", "mean": 0.08897508680820465, "median": 0.08751188218593597, "p05": 0.06392349302768707, "p95": 0.11800213158130646 }, { "label": "completed_group_3", "mean": 0.0967075377702713, "median": 0.09579810500144958, "p05": 0.06915527582168579, "p95": 0.12801720201969147 }, { "label": "completed_group_4", "mean": 0.09801244735717773, "median": 0.09654392302036285, "p05": 0.07431379705667496, "p95": 0.12754815816879272 }, { "label": "completed_group_5", "mean": 0.12438583374023438, "median": 0.12010829150676727, "p05": 0.08193086087703705, "p95": 0.1824759691953659 }, { "label": "completed_group_6", "mean": 0.1766587197780609, "median": 0.17520052194595337, "p05": 0.12216726690530777, "p95": 0.2341049611568451 }, { "label": "completed_group_7", "mean": 0.12203006446361542, "median": 0.1200856938958168, "p05": 0.05817953124642372, "p95": 0.19647708535194397 }, { "label": "current_group_partial", "mean": 0.11516246944665909, "median": 0.10430245846509933, "p05": 0.05224763602018356, "p95": 0.2294922173023224 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0835556983947754, "group": 8, "latest_source_mass": 0.09660109132528305, "layer": 32, "max_source_mass": 0.22511571645736694, "mixer_index": 63, "normalized_entropy": 0.9482670637704002, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09133295714855194, "median": 0.08019149303436279, "p05": 0.04441600665450096, "p95": 0.19931957125663757 }, { "label": "completed_group_1", "mean": 0.07054752856492996, "median": 0.06906121969223022, "p05": 0.05099350959062576, "p95": 0.09504306316375732 }, { "label": "completed_group_2", "mean": 0.09000501036643982, "median": 0.0901019275188446, "p05": 0.05998067185282707, "p95": 0.11849517375230789 }, { "label": "completed_group_3", "mean": 0.10956105589866638, "median": 0.1064089685678482, "p05": 0.0759081020951271, "p95": 0.1509535312652588 }, { "label": "completed_group_4", "mean": 0.07114207744598389, "median": 0.06995616853237152, "p05": 0.04991999268531799, "p95": 0.09545011073350906 }, { "label": "completed_group_5", "mean": 0.11565887182950974, "median": 0.11069735884666443, "p05": 0.06978912651538849, "p95": 0.17647549510002136 }, { "label": "completed_group_6", "mean": 0.22511571645736694, "median": 0.22372552752494812, "p05": 0.11895382404327393, "p95": 0.33412501215934753 }, { "label": "completed_group_7", "mean": 0.13003569841384888, "median": 0.12226496636867523, "p05": 0.06884703785181046, "p95": 0.21555939316749573 }, { "label": "current_group_partial", "mean": 0.09660109132528305, "median": 0.09000182151794434, "p05": 0.051288168877363205, "p95": 0.16580010950565338 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.679484486579895, "group": null, "latest_source_mass": 0.2827349305152893, "layer": null, "max_source_mass": 0.32717907428741455, "mixer_index": null, "normalized_entropy": 0.7643663300981273, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.08993907272815704, "median": 0.09290459752082825, "p05": 0.022419869899749756, "p95": 0.15114322304725647 }, { "label": "completed_group_1", "mean": 0.043493133038282394, "median": 0.042318835854530334, "p05": 0.026918282732367516, "p95": 0.06356139481067657 }, { "label": "completed_group_2", "mean": 0.04922422394156456, "median": 0.047891765832901, "p05": 0.03095712885260582, "p95": 0.07266643643379211 }, { "label": "completed_group_3", "mean": 0.037924449890851974, "median": 0.03562554344534874, "p05": 0.020909149199724197, "p95": 0.06224166974425316 }, { "label": "completed_group_4", "mean": 0.03939433768391609, "median": 0.036868683993816376, "p05": 0.020651251077651978, "p95": 0.06597590446472168 }, { "label": "completed_group_5", "mean": 0.03898489475250244, "median": 0.03362073004245758, "p05": 0.014276490546762943, "p95": 0.08282073587179184 }, { "label": "completed_group_6", "mean": 0.09112587571144104, "median": 0.07136045396327972, "p05": 0.02230823040008545, "p95": 0.22286513447761536 }, { "label": "completed_group_7", "mean": 0.32717907428741455, "median": 0.3213679790496826, "p05": 0.08271751552820206, "p95": 0.6128557920455933 }, { "label": "completed_group_8", "mean": 0.2827349305152893, "median": 0.2627606987953186, "p05": 0.05262754485011101, "p95": 0.571351170539856 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "optimizer_state_before": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08626964688301086, 0.16337208449840546, 0.15584443509578705, 0.04305505007505417, 0.1360139101743698, 0.03393339365720749, 0.03730965405702591, 0.037494905292987823, 0.05234000086784363, 0.04223497211933136, 0.041619230061769485, 0.05722156912088394, 0.062187470495700836, 0.0369485504925251, 0.04293901473283768, 0.045369699597358704, 0.04149825870990753, 0.02440357394516468, 0.026009807363152504, 0.03634416684508324, 0.038355328142642975, 0.02631118707358837, 0.033921729773283005, 0.02731483057141304, 0.027557097375392914, 0.024315863847732544, 0.02051021158695221, 0.018631720915436745, 0.019497457891702652, 0.021715763956308365, 0.017114246264100075, 0.020576251670718193 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9093876062470372e-05, "first_to_last_ratio": 1.111329265095317, "imbalance_abs_log_ratio": 0.105556834971714, "last_quartile_mean": 1.718111514037446e-05, "mean": 2.2651056013955895e-05, "normalized": [ 0.6946213574335215, 0.7243995962608978, 0.7139231072861242, 0.6747553054781453, 1.0542057875714943, 1.0725327169849952, 0.9486239566978842, 0.8605976469555432, 0.996006301084049, 0.884894598700678, 0.8197105185472539, 0.7590659103257471, 0.7393158988625451, 0.6909877240841125, 0.6603942706848981, 0.6306792317657164, 1.5905133022939153, 1.481373083704922, 1.3371031921307712, 1.1787516061453995, 2.2793598584591077, 2.032248538637951, 1.6978050641016875, 1.4100293011904417, 1.3387275181066927, 1.1259668340287685, 0.864980925975849, 0.7178664763990735, 0.7428917127279643, 0.519872888532167, 0.4082698409921403, 0.34952592784954284 ], "peak_layer": 21, "peak_normalized": 2.2793598584591077, "population_cv": 0.44327592901215446, "rest_mean": 1.9498221734937505e-05, "spike_contrast": 2.034870751806145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.96763611206552e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.5733907275716774e-05, 1.640841583139263e-05, 1.6171112292795442e-05, 1.528392022009939e-05, 2.3878874344518408e-05, 2.4293998649227433e-05, 2.1487334379344247e-05, 1.949344550666865e-05, 2.2560594516107813e-05, 2.004379712161608e-05, 1.856730887084268e-05, 1.719364445307292e-05, 1.674628583714366e-05, 1.565160164318513e-05, 1.4958627616579179e-05, 1.4285550605563913e-05, 3.602680590120144e-05, 3.355466469656676e-05, 3.028679930139333e-05, 2.6699968657339923e-05, 5.162990782991983e-05, 4.603257548296824e-05, 3.8457077607745305e-05, 3.193865268258378e-05, 3.0323592000058852e-05, 2.5504337827442214e-05, 1.9592731405282393e-05, 1.626043376745656e-05, 1.682728179730475e-05, 1.1775669918279164e-05, 9.247743037121836e-06, 7.917131370049901e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.440131457769894e-05, "first_to_last_ratio": 1.0844939416800765, "imbalance_abs_log_ratio": 0.08111346498986782, "last_quartile_mean": 6.860463827251806e-05, "mean": 8.996004032724159e-05, "normalized": [ 0.6841628567365404, 0.7116686494012134, 0.701182895977615, 0.6656603296641571, 1.0335676755092185, 1.041234037061157, 0.9271396156883384, 0.8517717767883587, 0.9923869547202551, 0.8818209991609433, 0.8176514654614152, 0.7595549596016796, 0.7388243939285175, 0.6938730529537547, 0.6653597800157137, 0.6317049728603569, 1.6039855928511222, 1.4970751654028693, 1.3561240124333966, 1.1902182609760135, 2.2883969934707804, 2.0368236734227096, 1.7055000379674186, 1.4234130023737248, 1.3429852370592492, 1.129666888881199, 0.8659251903055961, 0.7211434042764958, 0.7528108713654569, 0.5246536122144689, 0.41052437205164627, 0.35318926941861845 ], "peak_layer": 21, "peak_normalized": 2.2883969934707804, "population_cv": 0.4471671640899957, "rest_mean": 7.730859686961156e-05, "spike_contrast": 2.047351024432569, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015827783499844372, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.154731818241999e-05, 6.402174039976671e-05, 6.307844159891829e-05, 5.988283010083251e-05, 9.297978976974264e-05, 9.366945596411824e-05, 8.340551721630618e-05, 7.662542338948697e-05, 8.927517046686262e-05, 7.932865264592692e-05, 7.355595880653709e-05, 6.832959479652345e-05, 6.646467227255926e-05, 6.2420847825706e-05, 5.98557926423382e-05, 5.682820483343676e-05, 0.00014429460861720145, 0.000134676942252554, 0.00012199697084724903, 0.00010707208275562152, 0.00020586428581736982, 0.0001832327398005873, 0.00015342685219366103, 0.0001280502910958603, 0.00012081500608474016, 0.00010162487888010219, 7.789866504026577e-05, 6.487408973043784e-05, 6.772289634682238e-05, 4.7197860112646595e-05, 3.6930789065081626e-05, 3.177292092004791e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06597983045503497, "first_to_last_ratio": 1.0844939725271123, "imbalance_abs_log_ratio": 0.08111349343358171, "last_quartile_mean": 0.060839278157800436, "mean": 0.07977746380493045, "normalized": [ 0.6841628143469025, 0.7116686398363108, 0.7011829077449026, 0.665660355693298, 1.0335676288784994, 1.0412340885708804, 0.9271396405289938, 0.8517717921686788, 0.992386966395131, 0.8818210566482357, 0.817651474996915, 0.7595549913352903, 0.7388243852399965, 0.6938730655814884, 0.6653597733921379, 0.6317049688753786, 1.6039856083090063, 1.4970750632560381, 1.3561239578388844, 1.1902182174623175, 2.288397078979301, 2.0368237484113063, 1.705499960924184, 1.4234131140142534, 1.3429851911909423, 1.1296668304704789, 0.8659251700081085, 0.7211434030260389, 0.7528108714926084, 0.5246535886165229, 0.4105243987333961, 0.3531892470335724 ], "peak_layer": 21, "peak_normalized": 2.288397078979301, "population_cv": 0.4471671702125523, "rest_mean": 0.06855803672914151, "spike_contrast": 2.0473510723291724, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14036237001419066, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.054580774158239365, 0.05677511915564537, 0.05593859404325485, 0.053104694932699203, 0.08245540410280228, 0.08306701481342316, 0.07396484911441803, 0.06795219331979752, 0.0791701152920723, 0.07034944742918015, 0.06523016095161438, 0.06059537082910538, 0.05894153565168381, 0.055355433374643326, 0.053080715239048004, 0.050395820289850235, 0.1279619038105011, 0.11943285167217255, 0.10818812996149063, 0.09495259076356888, 0.18256251513957977, 0.16249263286590576, 0.1360604614019394, 0.11355628818273544, 0.107139952480793, 0.09012195467948914, 0.06908131390810013, 0.05753099173307419, 0.06005734205245972, 0.04185553267598152, 0.03275059536099434, 0.02817654237151146 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.814465545952771e-06, "first_to_last_ratio": 0.6078152624962909, "imbalance_abs_log_ratio": 0.49788428776455546, "last_quartile_mean": 4.630462115073897e-06, "mean": 4.80770270883113e-06, "normalized": [ 0.6119239482375798, 0.5829506456587372, 0.5668818808266815, 0.5828449444131751, 0.6017200654914986, 0.5893976649400692, 0.5746270627431066, 0.5729143242160235, 0.7060481880273346, 0.6743671364044601, 0.6598854036517451, 0.6543712037742013, 0.5975955414079575, 0.5980430336609813, 0.6043178095703504, 0.6380156504172737, 1.4746814450880759, 1.445384559546309, 1.4289995895563883, 1.3562595393822299, 2.503099703805365, 2.3685351606296248, 2.1182696084190358, 1.7837936000863488, 1.7255397755590118, 1.469090393074497, 1.1365579634387313, 0.9131688091869705, 0.9186131563845612, 0.6247692270757808, 0.4920086991179225, 0.42532426620797215 ], "peak_layer": 21, "peak_normalized": 2.503099703805365, "population_cv": 0.5770470220283189, "rest_mean": 3.82849157176679e-06, "spike_contrast": 2.6369244021398415, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0095442848978565e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.9419484235404525e-06, 2.802653398248367e-06, 2.725399554037722e-06, 2.8021452180837514e-06, 2.8928911888215225e-06, 2.833648750311113e-06, 2.762636086117709e-06, 2.7544017484615324e-06, 3.3944697861443274e-06, 3.242156708438415e-06, 3.1725328426546184e-06, 3.146022208966315e-06, 2.873061703212443e-06, 2.8752131129294867e-06, 2.9053803700662684e-06, 3.067389570787782e-06, 7.089829978212947e-06, 6.948979262233479e-06, 6.87020519762882e-06, 6.520492661366006e-06, 1.2034159226459451e-05, 1.1387212907720823e-05, 1.0184010534430854e-05, 8.575949323130772e-06, 8.295882253150921e-06, 7.062949862302048e-06, 5.464232799567981e-06, 4.390244157548295e-06, 4.416418960317969e-06, 3.0037047054065624e-06, 2.3654315555177163e-06, 2.04483262677968e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.492651735039544e-05, "first_to_last_ratio": 1.2798105542794473, "imbalance_abs_log_ratio": 0.24671206250860178, "last_quartile_mean": 4.2917693690469605e-05, "mean": 6.205384471513753e-05, "normalized": [ 0.760033268918417, 0.780541814588071, 0.7698447676581547, 0.7372284537013999, 1.0777402293794973, 1.0696059665333593, 0.9790150608789511, 0.9071332880771722, 1.0567179426445976, 0.9392066065134083, 0.8680723172882451, 0.7984285396279348, 0.8110573186244773, 0.7570450933968408, 0.7288656660723876, 0.6907977052301831, 1.5596824638835058, 1.458709216541477, 1.3045158420583727, 1.1442588808017629, 2.349062876038457, 2.014830196908582, 1.6040176787890326, 1.3006270514227598, 1.2844253595561577, 1.048991483387396, 0.8027663480449829, 0.6725642064369326, 0.6175576814869903, 0.43627687990216457, 0.3538390854052576, 0.3165407102030714 ], "peak_layer": 21, "peak_normalized": 2.349062876038457, "population_cv": 0.43649449327625967, "rest_mean": 5.3888103071518276e-05, "spike_contrast": 1.9698011906228117, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001061488495906815, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.7162986447801813e-05, 4.843562055611983e-05, 4.777182766702026e-05, 4.574785998556763e-05, 6.687792483717203e-05, 6.637316255364567e-05, 6.075164856156334e-05, 5.6291108194272965e-05, 6.557341112056747e-05, 5.8281380916014314e-05, 5.3867224778514355e-05, 4.95455606142059e-05, 5.032922490499914e-05, 4.697755866800435e-05, 4.5228916860651225e-05, 4.2866653529927135e-05, 9.678429341875017e-05, 9.051851520780474e-05, 8.095022349152714e-05, 7.100566290318966e-05, 0.00014576838293578476, 0.0001250279601663351, 9.953546395990998e-05, 8.070890908129513e-05, 7.97035318100825e-05, 6.509395461762324e-05, 4.981473830412142e-05, 4.173519482719712e-05, 3.8321828469634056e-05, 2.7072657758253627e-05, 2.195707565988414e-05, 1.9642568076960742e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.069607030032785e-05, "first_to_last_ratio": 1.664272059731362, "imbalance_abs_log_ratio": 0.5093878264694807, "last_quartile_mean": 2.4452775051031495e-05, "mean": 4.162627476489433e-05, "normalized": [ 0.8702598591224926, 0.8916298939131501, 0.8800748035118038, 0.8469315373702728, 1.1616929340311393, 1.145619284586338, 1.0550776811004199, 0.969941446038908, 1.1341420642114457, 0.9958805548845696, 0.9196031496339117, 0.8483230598856473, 0.9057150173064304, 0.8369896936785922, 0.8135149832683363, 0.764963852038374, 1.5382361125333859, 1.4503807190733784, 1.2717075500919934, 1.1291961376827389, 2.4272281265268307, 1.9471701631729643, 1.4094979938267411, 1.0867348620232957, 1.187654328911308, 0.9474334251489733, 0.7362053413370422, 0.6329446172788022, 0.43206791581619797, 0.292249639582794, 0.24209900469777865, 0.22883424771394378 ], "peak_layer": 21, "peak_normalized": 2.4272281265268307, "population_cv": 0.44080465046483597, "rest_mean": 3.6911273599360825e-05, "spike_contrast": 1.817528210674827, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.708728105877526e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.622567601269111e-05, 3.711523095262237e-05, 3.663423558464274e-05, 3.525460488162935e-05, 4.835694926441647e-05, 4.7687863116152585e-05, 4.391895345179364e-05, 4.037504913867451e-05, 4.721010918729007e-05, 4.1454797610640526e-05, 3.827965338132344e-05, 3.531252878019586e-05, 3.77015421690885e-05, 3.484076296444982e-05, 3.386359821888618e-05, 3.184259549016133e-05, 6.403103907359764e-05, 6.037394632585347e-05, 5.293644790071994e-05, 4.700422869063914e-05, 0.00010103646491188556, 8.105344022624195e-05, 5.867215077159926e-05, 4.5236723963171244e-05, 4.943762542097829e-05, 3.943812407669611e-05, 3.064548582187854e-05, 2.634712654980831e-05, 1.798537778086029e-05, 1.216526379721472e-05, 1.0077679689857177e-05, 9.525517270958517e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014326406108011724, "first_to_last_ratio": 0.9675119405224295, "imbalance_abs_log_ratio": 0.03302751252910728, "last_quartile_mean": 0.00014807472143729683, "mean": 0.0001869507727860764, "normalized": [ 0.6324253729661448, 0.6595289911573035, 0.6411507649026799, 0.6215886971725172, 0.9471130335994487, 0.9408601337761202, 0.8694437233530738, 0.8184469243209181, 0.9716541786960263, 0.8739107809137097, 0.8227884288340134, 0.7600796243144189, 0.7142362602596459, 0.6659890963200943, 0.6364485580818953, 0.6213951524405211, 1.57652704046099, 1.4727978057853581, 1.3754263968525124, 1.1477387862326969, 2.376960332745933, 2.1664621252591743, 1.8156424692300108, 1.5349698379454737, 1.4342006350688952, 1.1801289816751488, 0.9093707600442614, 0.7581683846806241, 0.752048199021001, 0.5241419408435032, 0.417566045735057, 0.36079053731082966 ], "peak_layer": 21, "peak_normalized": 2.376960332745933, "population_cv": 0.4810121978731533, "rest_mean": 0.00015698162638323588, "spike_contrast": 2.2218151983591756, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00034878416336141526, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011823241220554337, 0.00012329945457167923, 0.00011986363097094, 0.00011620648729149252, 0.00017706351354718208, 0.00017589452909305692, 0.0001625431759748608, 0.00015300928498618305, 0.0001816514995880425, 0.00016337829583790153, 0.00015382093260996044, 0.00014209747314453125, 0.000133527020807378, 0.0001245071762241423, 0.00011898454977199435, 0.00011617030395427719, 0.000294732948532328, 0.0002753406879492104, 0.0002571370278019458, 0.00021457065304275602, 0.0004443745710887015, 0.0004050217685289681, 0.0003394357627257705, 0.0002869637974072248, 0.0002681249170564115, 0.0002206260251114145, 0.00017000756633933634, 0.00014174016541801393, 0.00014059599197935313, 9.798874089028686e-05, 7.806429493939504e-05, 6.745006976416335e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13547764718532562, 0.039667773991823196, 0.026703152805566788, 0.05304057523608208, 0.016798952594399452, 0.00753376679494977, 0.01294450368732214, 0.03526800498366356, 0.030294200405478477, 0.034149833023548126, 0.04367009550333023, 0.05930330231785774, 0.04527241364121437, 0.04281846433877945, 0.04812675341963768, 0.06094835698604584, 0.038031257688999176, 0.040106989443302155, 0.04961257055401802, 0.07085621356964111, 0.03753058984875679, 0.04544757679104805, 0.05368982255458832, 0.06174425780773163, 0.05051252990961075, 0.054937634617090225, 0.05777662619948387, 0.07133159041404724, 0.0636468306183815, 0.09261580556631088, 0.1020733118057251, 0.12878532707691193 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.8983760583068943e-05, "first_to_last_ratio": 1.3500112943631184, "imbalance_abs_log_ratio": 0.3001129586102442, "last_quartile_mean": 1.4061927231523441e-05, "mean": 2.0583363209425443e-05, "normalized": [ 0.7641682400188751, 0.7964973937008093, 0.7903246216156576, 0.7412309789481601, 1.1595280618590635, 1.1689606938823263, 1.0270355404284468, 0.9305474685745297, 1.009443360915331, 0.92439237085469, 0.8548158178555026, 0.798458806391288, 0.7719145608477377, 0.7216837495160779, 0.7013551805794737, 0.6759138154313806, 1.6233726027813016, 1.4876771328688778, 1.307017635744062, 1.1529668955961847, 2.3147875877516704, 1.9234371956824674, 1.5697411551918221, 1.319372635699807, 1.2524447630862485, 0.9556486905621968, 0.7985175736559498, 0.6718204766110775, 0.5751452774144216, 0.45005871272267506, 0.38512561316412136, 0.3765953900477665 ], "peak_layer": 21, "peak_normalized": 2.3147875877516704, "population_cv": 0.4284577654113479, "rest_mean": 1.8006796246347086e-05, "spike_contrast": 1.9157669325572948, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4496824810048565e-05, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.5729152437415905e-05, 1.639459514990449e-05, 1.626753874006681e-05, 1.5257026461767964e-05, 2.3866987248766236e-05, 2.4061142539721914e-05, 2.1139845557627268e-05, 1.9153796529280953e-05, 2.0777739337063394e-05, 1.9027103917323984e-05, 1.7594984456081875e-05, 1.643496761971619e-05, 1.588859777257312e-05, 1.4854678738629445e-05, 1.4436248420679476e-05, 1.391257956129266e-05, 3.341446790727787e-05, 3.062139876419678e-05, 2.690281871764455e-05, 2.3731936380499974e-05, 4.76461136713624e-05, 3.9590806409250945e-05, 3.2310552342096344e-05, 2.7157126169186085e-05, 2.5779525458347052e-05, 1.9670464098453522e-05, 1.6436177247669548e-05, 1.3828324881615117e-05, 1.1838424143206794e-05, 9.263721949537285e-06, 7.92718037700979e-06, 7.751599696348421e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.388659878415638e-05, "first_to_last_ratio": 1.3167747149325586, "imbalance_abs_log_ratio": 0.2751853488521164, "last_quartile_mean": 5.611179949482903e-05, "mean": 8.168216641024628e-05, "normalized": [ 0.7533112744592003, 0.7827721531209632, 0.7759215503905756, 0.7316303776947085, 1.1365209343706772, 1.135346283191908, 1.004240264612514, 0.9167546285036348, 1.0078386860630901, 0.9220221645206613, 0.8550885147558498, 0.8013979502567415, 0.7721731241940338, 0.7274558542877592, 0.7065441772286032, 0.6779218197522006, 1.6398171430789432, 1.5049144188657668, 1.3267658786647274, 1.1597753221298892, 2.322010757724194, 1.9301972923447444, 1.5840545712188465, 1.3299018151740607, 1.2567171362804614, 0.9579514188552475, 0.8014611945379978, 0.6752124703737634, 0.5811954370019358, 0.4526517471161714, 0.3896333192051585, 0.38080032002497083 ], "peak_layer": 21, "peak_normalized": 2.322010757724194, "population_cv": 0.4315619777351012, "rest_mean": 7.13270411425684e-05, "spike_contrast": 1.929139925777552, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001375998428557068, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.153209687909111e-05, 6.39385252725333e-05, 6.337895320029929e-05, 5.976115426165052e-05, 9.283349208999425e-05, 9.273754403693601e-05, 8.202852040994912e-05, 7.48825041227974e-05, 8.232244726968929e-05, 7.531276787631214e-05, 6.984548235777766e-05, 6.545992073370144e-05, 6.307277362793684e-05, 5.942017014604062e-05, 5.77120590605773e-05, 5.5374122894136235e-05, 0.00013394381676334888, 0.00012292466999497265, 0.00010837311128852889, 9.473296086071059e-05, 0.0001896668691188097, 0.0001576626964379102, 0.00012938900908920914, 0.00010862926137633622, 0.0001026513782562688, 7.824754720786586e-05, 6.546508666360751e-05, 5.515281736734323e-05, 4.747330240206793e-05, 3.697357533383183e-05, 3.182609361829236e-05, 3.1104595109354705e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06552337622269988, "first_to_last_ratio": 1.3167747352194785, "imbalance_abs_log_ratio": 0.2751853642586392, "last_quartile_mean": 0.0497605053242296, "mean": 0.07243656268110499, "normalized": [ 0.7533112993917941, 0.7827721634134585, 0.7759215519760847, 0.7316303564434335, 1.1365209991517615, 1.1353462736009083, 1.004240278158844, 0.9167546473202071, 1.0078387172969752, 0.9220222438210248, 0.8550884472945091, 0.8013979045056098, 0.7721731478740148, 0.7274558173306654, 0.7065441905330453, 0.6779218137445964, 1.6398170980615752, 1.5049143559021794, 1.3267658529793183, 1.1597753399419397, 2.322010804717546, 1.9301971994356375, 1.584054611353896, 1.3299018487169911, 1.2567171977522282, 0.9579514011929847, 0.8014611613319625, 0.6752125187290771, 0.5811954405704202, 0.4526517065938406, 0.3896333019053031, 0.3808003089581684 ], "peak_layer": 21, "peak_normalized": 2.322010804717546, "population_cv": 0.43156197832928544, "rest_mean": 0.0632535337160031, "spike_contrast": 1.9291399535166673, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12202491909265518, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.054567281156778336, 0.05670132488012314, 0.05620509013533592, 0.052996788173913956, 0.08232567459344864, 0.08224058151245117, 0.07274371385574341, 0.06640655547380447, 0.07300437241792679, 0.06678812205791473, 0.06193966791033745, 0.05805050954222679, 0.05593356862664223, 0.052694398909807205, 0.05117963254451752, 0.04910632595419884, 0.11878271400928497, 0.10901082307100296, 0.09610635787248611, 0.08401013910770416, 0.16819848120212555, 0.13981685042381287, 0.11474347114562988, 0.09633351862430573, 0.09103227406740189, 0.06939070671796799, 0.0580550916492939, 0.048910073935985565, 0.04209979996085167, 0.032788533717393875, 0.02822369709610939, 0.027583865448832512 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7909886171073595e-06, "first_to_last_ratio": 0.73274866566625, "imbalance_abs_log_ratio": 0.3109525203873344, "last_quartile_mean": 3.8089303302513144e-06, "mean": 4.300410559210377e-06, "normalized": [ 0.6752193249648764, 0.648060123214754, 0.6340762852531016, 0.648956154331942, 0.6688990988684992, 0.6540429695649356, 0.6333917443388257, 0.6293955312096993, 0.7505991389397793, 0.716809349220506, 0.6944304605268068, 0.6987346033913733, 0.6285871627816183, 0.6319728565674881, 0.6466036959323995, 0.7005312655376627, 1.4911248937697517, 1.437684597591998, 1.3858356594460568, 1.3108305532381637, 2.5649491626046244, 2.293141742331734, 1.9862148934338621, 1.784203258435835, 1.6977937545261073, 1.262754388821972, 1.06360231285752, 0.8741748207857217, 0.7144275994278602, 0.5384026644950104, 0.4702436547730354, 0.4643062788164799 ], "peak_layer": 21, "peak_normalized": 2.5649491626046244, "population_cv": 0.5484988453962795, "rest_mean": 3.4520665276768676e-06, "spike_contrast": 2.5727975571398605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.881468329491326e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.9037203148618573e-06, 2.786924596875906e-06, 2.726788352447329e-06, 2.7907778985536424e-06, 2.8765407478203997e-06, 2.8126532924943604e-06, 2.7238445454713656e-06, 2.706659188334015e-06, 3.2278844628308434e-06, 3.0825744943285827e-06, 2.986336085086805e-06, 3.0048456665099366e-06, 2.7031828722101636e-06, 2.717742745517171e-06, 2.7806613616121467e-06, 3.012572051375173e-06, 6.412449238268891e-06, 6.18263402429875e-06, 5.959662303212099e-06, 5.637109552480979e-06, 1.103033446270274e-05, 9.86145096248947e-06, 8.541539500583895e-06, 7.672806532355025e-06, 7.301210189325502e-06, 5.430362307379255e-06, 4.573926617013058e-06, 3.7593106299027568e-06, 3.0723319923708914e-06, 2.3153525035013445e-06, 2.0222407783876406e-06, 1.9967076241300674e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.4603370244876714e-05, "first_to_last_ratio": 1.5303484438582295, "imbalance_abs_log_ratio": 0.42549545055909027, "last_quartile_mean": 3.5680351402334054e-05, "mean": 5.699801209857469e-05, "normalized": [ 0.8264909998027334, 0.8487922685363437, 0.841606882150277, 0.8012941819001962, 1.1726168644790063, 1.1546993854462606, 1.050180566275997, 0.9682170449733281, 1.0710784836103833, 0.9771184045421, 0.8995231631213109, 0.8346230595845928, 0.8429727036171436, 0.7876480304240737, 0.76714462416435, 0.7352965831476843, 1.5890796305970005, 1.4640441930989896, 1.2783685546924959, 1.1236995336661568, 2.371393623645145, 1.8882377423241299, 1.482591511200619, 1.2153386927849252, 1.1838783972428712, 0.8905908477409856, 0.7420819055609336, 0.634319683256865, 0.48200464426330825, 0.3870264062137896, 0.3477017707900011, 0.34033961714600236 ], "peak_layer": 21, "peak_normalized": 2.371393623645145, "population_cv": 0.4245100179440155, "rest_mean": 5.036631457052728e-05, "spike_contrast": 1.8426835384207285, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.28091787500307e-05, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.7108344006119296e-05, 4.8379471991211176e-05, 4.796991925104521e-05, 4.567217547446489e-05, 6.683683022856712e-05, 6.581556954188272e-05, 5.98582046222873e-05, 5.518644684343599e-05, 6.104934436734766e-05, 5.569380664383061e-05, 5.1271032134536654e-05, 4.7571855247952044e-05, 4.804776835953817e-05, 4.489437196752988e-05, 4.372571856947616e-05, 4.191044354229234e-05, 9.057438001036644e-05, 8.344760863110423e-05, 7.286446634680033e-05, 6.404863961506635e-05, 0.00013516472245100886, 0.00010762579768197611, 8.450476889265701e-05, 6.927188951522112e-05, 6.747871520929039e-05, 5.076190791442059e-05, 4.2297193431295455e-05, 3.615496098063886e-05, 2.7473306545289233e-05, 2.205973578384146e-05, 1.9818309738184325e-05, 1.939868161571212e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.048281562063494e-05, "first_to_last_ratio": 1.9630119222573643, "imbalance_abs_log_ratio": 0.6744799887640945, "last_quartile_mean": 2.062280680092954e-05, "mean": 3.8703238459447675e-05, "normalized": [ 0.9360916836648231, 0.957580281730348, 0.9482712191433338, 0.910353586321295, 1.2460232346920597, 1.2224003463169428, 1.1176779980540072, 1.0294421165646366, 1.1460215914378586, 1.0320514664718978, 0.9473985532737729, 0.8788138212913126, 0.9341792256960503, 0.8671852947881752, 0.8459732351746374, 0.8066658603360692, 1.5684062538088193, 1.4484147907409404, 1.2467146748187818, 1.105475527367378, 2.4247996872064124, 1.8050505092184956, 1.2955371531304132, 1.0167160839549727, 1.0675715240747539, 0.8180409715888471, 0.6887416678917638, 0.6147875592971825, 0.3233626767922262, 0.26293806156630717, 0.24647344903818308, 0.24083989454730284 ], "peak_layer": 21, "peak_normalized": 2.4247996872064124, "population_cv": 0.4349671070041115, "rest_mean": 3.4962391341482374e-05, "spike_contrast": 1.684776430797849, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.89038128964603e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.622977965278551e-05, 3.7061457987874746e-05, 3.670116711873561e-05, 3.5233631933806464e-05, 4.822513437829912e-05, 4.7310852096416056e-05, 4.3257758079562336e-05, 3.984274371759966e-05, 4.435474693309516e-05, 3.9943734009284526e-05, 3.666739212349057e-05, 3.4012940886896104e-05, 3.615576133597642e-05, 3.3562879252713174e-05, 3.27419038512744e-05, 3.12205811496824e-05, 6.070240124245174e-05, 5.605834303423762e-05, 4.8251895350404084e-05, 4.2785482946783304e-05, 9.384760051034391e-05, 6.986130028963089e-05, 5.014148337068036e-05, 3.935020504286513e-05, 4.1318475268781185e-05, 3.166083479300141e-05, 2.665653300937265e-05, 2.3794269509380683e-05, 1.2515182788774837e-05, 1.017655449686572e-05, 9.539320672047324e-06, 9.321283869212493e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014194276809575967, "first_to_last_ratio": 1.1782647028111788, "imbalance_abs_log_ratio": 0.16404276525959405, "last_quartile_mean": 0.00012046764004480792, "mean": 0.00016912808678171132, "normalized": [ 0.6984404643238884, 0.7279605452185844, 0.716541899087504, 0.6844000690950613, 1.0470185740245912, 1.0262206207114242, 0.9392540256525687, 0.8742594105695676, 1.0007921238341297, 0.9198815102082726, 0.860099074470987, 0.8041171467379208, 0.7517592573362873, 0.7071749387596568, 0.6728971890246059, 0.6653759328341454, 1.636701579189545, 1.4903312139370086, 1.3329613072711604, 1.154725819226416, 2.4301173158726157, 2.015490849459162, 1.6993053786715127, 1.4458823665938263, 1.3104146608182012, 1.004154426063848, 0.8373805185074344, 0.7042667168313188, 0.5859481462881533, 0.463333171612487, 0.3998633715178443, 0.3929303762502709 ], "peak_layer": 21, "peak_normalized": 2.4301173158726157, "population_cv": 0.4589844309602813, "rest_mean": 0.00014469089122333875, "spike_contrast": 2.0809115228419945, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00030108894279692324, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.00011812589946202934, 0.00012311857426539063, 0.00012118736049160361, 0.00011575127427931875, 0.0001770802482496947, 0.0001735627301968634, 0.00015885423636063933, 0.0001478618214605376, 0.00016926205717027187, 0.0001555777998873964, 0.00014546691090799868, 0.00013599879457615316, 0.00012714360491372645, 0.00011960314441239461, 0.00011380581418052316, 0.00011253375851083547, 0.0002768122067209333, 0.00025205686688423157, 0.00022544119565282017, 0.00019529656856320798, 0.0004110010922886431, 0.0003408761112950742, 0.0002874002675525844, 0.0002445393183734268, 0.00022162792447488755, 0.000169830716913566, 0.00014162456500343978, 0.0001191112824017182, 9.910028893500566e-05, 7.836265285732225e-05, 6.762812699889764e-05, 6.64555627736263e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08626964688301086, 0.27676042914390564, 0.41335058212280273, 0.43042075634002686, 0.1360139101743698, 0.15445689857006073, 0.1670466512441635, 0.17834730446338654, 0.05234000086784363, 0.08269349485635757, 0.11548293381929398, 0.16410532593727112, 0.062187470495700836, 0.0988159254193306, 0.13199056684970856, 0.17199495434761047, 0.04149825870990753, 0.06434677541255951, 0.08728200942277908, 0.11647984385490417, 0.038355328142642975, 0.06353354454040527, 0.09196189045906067, 0.1169828325510025, 0.027557097375392914, 0.06579430401325226, 0.09665275365114212, 0.12799666821956635, 0.019497457891702652, 0.07176875323057175, 0.14131556451320648, 0.22250843048095703 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9093950641035917e-05, "first_to_last_ratio": 1.1113330910687114, "imbalance_abs_log_ratio": 0.10556027766590795, "last_quartile_mean": 1.71811230984531e-05, "mean": 2.265107818288925e-05, "normalized": [ 0.6945983528743319, 0.724410451166159, 0.7139225691689148, 0.6747768091993389, 1.054240973252137, 1.0725014726920594, 0.9486330663611198, 0.8605955198001844, 0.995990630507342, 0.8848804020515041, 0.8196954220375458, 0.7590673356456394, 0.7393107585217125, 0.6909917857854257, 0.6603937849571249, 0.6306897768726338, 1.5905019484551022, 1.4813909070064188, 1.337087187717671, 1.1787423417035985, 2.2793733673501118, 2.0322272765048472, 1.6978229967971588, 1.4100558672312264, 1.3387234775131232, 1.1259802671905756, 0.8649860219589377, 0.7178508371275001, 0.7428919493063704, 0.5198747888679629, 0.4082655064793689, 0.3495261478968529 ], "peak_layer": 21, "peak_normalized": 2.2793733673501118, "population_cv": 0.44327706814451906, "rest_mean": 1.9498213381060243e-05, "spike_contrast": 2.034881213850395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.9676548112765886e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.573340159666259e-05, 1.640867776586674e-05, 1.617111593077425e-05, 1.5284422261174768e-05, 2.3879694708739407e-05, 2.42933147092117e-05, 2.148756175301969e-05, 1.9493416402838193e-05, 2.2560261641046964e-05, 2.0043495169375092e-05, 1.856698509072885e-05, 1.7193693565786816e-05, 1.6746185792726465e-05, 1.565170896355994e-05, 1.4958631254557986e-05, 1.4285803445091005e-05, 3.602658398449421e-05, 3.355510125402361e-05, 3.028646642633248e-05, 2.6699784939410165e-05, 5.1630264351842925e-05, 4.603213892551139e-05, 3.845752144115977e-05, 3.193928569089621e-05, 3.032353015441913e-05, 2.5504667064524256e-05, 1.9592866010498255e-05, 1.6260095435427502e-05, 1.6827303625177592e-05, 1.177572448796127e-05, 9.247653906641062e-06, 7.917144102975726e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.440197032337892e-05, "first_to_last_ratio": 1.0845009049103946, "imbalance_abs_log_ratio": 0.08111988568775846, "last_quartile_mean": 6.860480243631173e-05, "mean": 8.99602513300124e-05, "normalized": [ 0.6841490391911266, 0.7116943175111012, 0.70118634676606, 0.6656715473421221, 1.0336051249539524, 1.0412072500471912, 0.9271427791318646, 0.8517742273313543, 0.9923652968187024, 0.8818152912537373, 0.8176321585165575, 0.7595628027406727, 0.7388217713310421, 0.693873609218528, 0.665364325823853, 0.6317142481886351, 1.6039671105807234, 1.4970661541772026, 1.3561211551482075, 1.1902099694866315, 2.2884215514970006, 2.0367978673096307, 1.7055055815020073, 1.4234313395007192, 1.3429770725313894, 1.12966876849781, 0.8659356147413246, 0.7211347571729164, 0.7528133113832752, 0.5246594990466679, 0.4105183137414301, 0.3531917975165635 ], "peak_layer": 21, "peak_normalized": 2.2884215514970006, "population_cv": 0.44716612344844425, "rest_mean": 7.73087299926879e-05, "spike_contrast": 2.0473556681960123, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015827846655156464, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.154621951282024e-05, 6.402419967344031e-05, 6.307889998424798e-05, 5.988397970213555e-05, 9.298337681684643e-05, 9.366726590087637e-05, 8.340599742950872e-05, 7.662582356715575e-05, 8.927343151299283e-05, 7.932832522783428e-05, 7.355439447565004e-05, 6.833046063547954e-05, 6.64645922370255e-05, 6.242104427656159e-05, 5.985634197713807e-05, 5.6829172535799444e-05, 0.00014429328439291567, 0.00013467644748743623, 0.00012199699995107949, 0.00010707158799050376, 0.00020586697792168707, 0.0001832308480516076, 0.0001534277107566595, 0.0001280522410525009, 0.00012081455497536808, 0.00010162528633372858, 7.789978553773835e-05, 6.487346399808303e-05, 6.772327469661832e-05, 4.719850039691664e-05, 3.693033067975193e-05, 3.17732228722889e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06598041160032153, "first_to_last_ratio": 1.0845009182393224, "imbalance_abs_log_ratio": 0.08111989797813793, "last_quartile_mean": 0.06083942437544465, "mean": 0.07977765158284456, "normalized": [ 0.6841490163544541, 0.7116942818442077, 0.7011863471790457, 0.6656715368689867, 1.0336051678040334, 1.0412072624828899, 0.9271427815934556, 0.8517741767131453, 0.992365298440462, 0.8818153387666878, 0.8176321795570402, 0.7595627761828878, 0.7388218056939301, 0.6938736270746664, 0.665364324455644, 0.6317142220498281, 1.6039670769938226, 1.497066216157726, 1.3561211394078925, 1.190209999241352, 2.2884215780128287, 2.0367978476522746, 1.7055056593265296, 1.423431337144962, 1.3429770803514192, 1.1296687476987397, 0.8659355529359181, 0.721134747932228, 0.7528132554883469, 0.5246594981795372, 0.41051829590461564, 0.35319182451044234 ], "peak_layer": 21, "peak_normalized": 2.2884215780128287, "population_cv": 0.4471661317726333, "rest_mean": 0.06855815552450993, "spike_contrast": 2.047355697130315, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14036293029785157, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.054579801857471466, 0.05677729845046997, 0.055939000099897385, 0.053105711936950684, 0.08245859295129776, 0.08306507021188736, 0.07396527379751205, 0.06795254349708557, 0.07916857302188873, 0.07034915685653687, 0.06522877514362335, 0.06059613451361656, 0.058941468596458435, 0.055355608463287354, 0.0530812032520771, 0.05039667710661888, 0.12796072661876678, 0.11943242698907852, 0.10818815976381302, 0.09495215862989426, 0.1825648993253708, 0.16249094903469086, 0.13606123626232147, 0.11355800926685333, 0.10713955760002136, 0.09012231975793839, 0.06908230483531952, 0.05753043666481972, 0.06005767360329628, 0.041856102645397186, 0.03275018557906151, 0.028176814317703247 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8144915802386095e-06, "first_to_last_ratio": 0.6078045854813123, "imbalance_abs_log_ratio": 0.49790185413614213, "last_quartile_mean": 4.630586289522398e-06, "mean": 4.807722199018372e-06, "normalized": [ 0.6119714094009943, 0.5829248721673723, 0.5668443018259814, 0.5828395075254097, 0.6017526232967554, 0.589367845371327, 0.5746677702647779, 0.5729165418286368, 0.7059387263625273, 0.6743568356158998, 0.6598434749727681, 0.6543816039835032, 0.5975752891719487, 0.5980213135145247, 0.6043413237993088, 0.6380324542505855, 1.4745083318439967, 1.4453097462314772, 1.4289980528954762, 1.3562696480279695, 2.5032567859750894, 2.3684498892562993, 2.118380011365409, 1.783803961861413, 1.725559075482424, 1.4691197183784992, 1.1366835074286532, 0.9131953750566115, 0.9185830426535001, 0.624807082892335, 0.49197761908752924, 0.4253222582109956 ], "peak_layer": 21, "peak_normalized": 2.5032567859750894, "population_cv": 0.5770592537991902, "rest_mean": 3.8284693649378214e-06, "spike_contrast": 2.6370036013641465, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0095687503053342e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.9421885301417205e-06, 2.8025408482790226e-06, 2.725229933275841e-06, 2.8021304387948476e-06, 2.8930594453413505e-06, 2.833516873579356e-06, 2.7628429961623624e-06, 2.754423576334375e-06, 3.3939572858798783e-06, 3.242120328650344e-06, 3.1723441225040006e-06, 3.146084964100737e-06, 2.8729759833368007e-06, 2.875120344469906e-06, 2.9055051982140867e-06, 3.0674827939947136e-06, 7.089026439643931e-06, 6.948647751414683e-06, 6.87022566125961e-06, 6.520567694678903e-06, 1.2034963219775818e-05, 1.1386849109840114e-05, 1.018458260659827e-05, 8.576033906138036e-06, 8.296008672914468e-06, 7.063119483063929e-06, 5.4648585319228005e-06, 4.390389676700579e-06, 4.416292085807072e-06, 3.003898882525391e-06, 2.3652917207073187e-06, 2.0448312625376275e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.492669151863083e-05, "first_to_last_ratio": 1.2798129309515296, "imbalance_abs_log_ratio": 0.2467139195567934, "last_quartile_mean": 4.2917750079141115e-05, "mean": 6.205391622415846e-05, "normalized": [ 0.7600355002625492, 0.7805513505607012, 0.7698486878525821, 0.7372280145247141, 1.0777481330957353, 1.0696009818823022, 0.9790057250388184, 0.9071387502213032, 1.0567092207752022, 0.9392041171743681, 0.8680643404431132, 0.7984310198570167, 0.8110583772742559, 0.7570453348972137, 0.7288688127245345, 0.6908032407944877, 1.5596744521873913, 1.4587063630480184, 1.3045232499410013, 1.144251230574794, 2.3490749428268014, 2.0148170878788103, 1.604017002890392, 1.3006314152297547, 1.2844195410928985, 1.0489976614499437, 0.8027713441974065, 0.6725629037598078, 0.6175554455540694, 0.436277197915466, 0.3538367723039011, 0.316541781770646 ], "peak_layer": 21, "peak_normalized": 2.3490749428268014, "population_cv": 0.4364939701143597, "rest_mean": 5.3888172462595525e-05, "spike_contrast": 1.9698001933592688, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010614893253659829, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.716317926067859e-05, 4.843626811634749e-05, 4.777212598128244e-05, 4.574788545141928e-05, 6.687849236186594e-05, 6.637292972300202e-05, 6.075113924453035e-05, 5.629151200992055e-05, 6.557294545928016e-05, 5.8281293604522943e-05, 5.386679185903631e-05, 4.954577161697671e-05, 5.032934859627858e-05, 4.6977627789601684e-05, 4.522916424321011e-05, 4.28670464316383e-05, 9.678390779299662e-05, 9.05184424482286e-05, 8.095077646430582e-05, 7.100527000147849e-05, 0.00014576929970644414, 0.0001250272907782346, 9.953553671948612e-05, 8.070927287917584e-05, 7.970326259965077e-05, 6.509441300295293e-05, 4.9815105739980936e-05, 4.1735162085387856e-05, 3.832173388218507e-05, 2.7072708689956926e-05, 2.195695742557291e-05, 1.9642657207441516e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.069824126418098e-05, "first_to_last_ratio": 1.664243286286015, "imbalance_abs_log_ratio": 0.5093705374127352, "last_quartile_mean": 2.4454502295157e-05, "mean": 4.162822952480383e-05, "normalized": [ 0.8703493828745057, 0.8916901865290843, 0.88010828499616, 0.8470021437701022, 1.1617085596346044, 1.1457007720909245, 1.0547659609018474, 0.9699520930859898, 1.1341637901024582, 0.9959865521758748, 0.9196947259511007, 0.8482508896276112, 0.9057174940795618, 0.8369874449429452, 0.8134828126487191, 0.7649775699240196, 1.5380296464446301, 1.4500791010223821, 1.2716272092758651, 1.129435876960094, 2.4274111082159515, 1.9474132658179035, 1.4092447881310306, 1.0866205597228813, 1.1876165623415504, 0.9475667789345992, 0.7362149039491379, 0.6328866680951589, 0.43209730937738905, 0.2922508384567999, 0.2421482427484381, 0.22881847717067896 ], "peak_layer": 21, "peak_normalized": 2.4274111082159515, "population_cv": 0.44080997556292906, "rest_mean": 3.691297485972798e-05, "spike_contrast": 1.8175344840442327, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.709060471621342e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.623110387707129e-05, 3.711948374984786e-05, 3.663734969450161e-05, 3.5259199648862705e-05, 4.8359870561398566e-05, 4.769349470734596e-05, 4.3908039515372366e-05, 4.037738835904747e-05, 4.721323057310656e-05, 4.146115679759532e-05, 3.828526314464398e-05, 3.531118272803724e-05, 3.770341572817415e-05, 3.484230546746403e-05, 3.386384923942387e-05, 3.184466186212376e-05, 6.402545113814995e-05, 6.036422564648092e-05, 5.293558933772147e-05, 4.7016415919642895e-05, 0.00010104882676387206, 8.10673664091155e-05, 5.866436549695209e-05, 4.523409006651491e-05, 4.9438374844612554e-05, 3.9445527363568544e-05, 3.064732300117612e-05, 2.6345951482653618e-05, 1.798744597181212e-05, 1.2165884982096031e-05, 1.0080202628159896e-05, 9.525308087177109e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014326045857160352, "first_to_last_ratio": 0.9674628975633102, "imbalance_abs_log_ratio": 0.03307820358530557, "last_quartile_mean": 0.0001480785040257615, "mean": 0.00018694848358791205, "normalized": [ 0.6324557293416971, 0.6595615864784095, 0.6410845907888777, 0.6214095723982178, 0.9472452039473033, 0.9409894253465454, 0.8694511005112591, 0.8182813410919756, 0.9717054632953117, 0.8737801260796588, 0.8227731283696897, 0.7600623105493365, 0.714347909521877, 0.6659205019824871, 0.636328539521148, 0.6214749962225501, 1.5765359147034264, 1.472710913100365, 1.3755308859596458, 1.1476507932250721, 2.3770794208861905, 2.166537848106843, 1.8154778878942872, 1.5349498698581336, 1.4342364113292647, 1.1800424750363023, 0.9095504950044728, 0.7581051224006025, 0.7520703292100521, 0.5242482655784406, 0.4175420470120725, 0.3608597952484843 ], "peak_layer": 21, "peak_normalized": 2.3770794208861905, "population_cv": 0.4810200206149476, "rest_mean": 0.0001569793853882907, "spike_contrast": 2.2218306754300956, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00034878161386586726, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011823663953691721, 0.00012330403842497617, 0.00011984979209955782, 0.00011617157724685967, 0.00017708605446387082, 0.00017591654614079744, 0.0001625425647944212, 0.00015297645586542785, 0.00018165886285714805, 0.00016335186955984682, 0.000153816188685596, 0.00014209249638952315, 0.0001335462584393099, 0.00012449282803572714, 0.00011896065552718937, 0.00011618380813160911, 0.0002947309985756874, 0.00027532107196748257, 0.00025715341325849295, 0.00021455157548189163, 0.0004443913931027055, 0.0004050309653393924, 0.0003394008381292224, 0.00028695655055344105, 0.00026812832220457494, 0.0002206071512773633, 0.00017003908578772098, 0.0001417266030330211, 0.00014059840759728104, 9.800741827348247e-05, 7.805885252309963e-05, 6.746219150954857e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16424456238746643, 0.29588937759399414, 0.42337876558303833, 0.45154979825019836, 0.13891471922397614, 0.1555756777524948, 0.16914354264736176, 0.1874145120382309, 0.06211879849433899, 0.09842939674854279, 0.137006014585495, 0.20118924975395203, 0.08399304747581482, 0.12138351052999496, 0.1587214469909668, 0.2071460783481598, 0.0581100657582283, 0.08358953893184662, 0.10947012156248093, 0.1472771316766739, 0.05584336072206497, 0.08318652212619781, 0.11238116025924683, 0.1379220336675644, 0.05995604768395424, 0.09392217546701431, 0.1259489357471466, 0.17086055874824524, 0.06700707972049713, 0.13971327245235443, 0.2211977243423462, 0.32473331689834595 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.8983928498528257e-05, "first_to_last_ratio": 1.3500192298694966, "imbalance_abs_log_ratio": 0.3001188366966636, "last_quartile_mean": 1.4061968954592885e-05, "mean": 2.0583478459457183e-05, "normalized": [ 0.7642043470230905, 0.7964604133404727, 0.7903456474112416, 0.7412458727049767, 1.1595129974656138, 1.169001073870473, 1.0269918785986678, 0.9305547186552634, 1.0094506994739345, 0.9243807439337586, 0.8548226966273005, 0.7984573403240672, 0.7719215503109763, 0.72169521786754, 0.7013534186825957, 0.6759055681347007, 1.6233704062346581, 1.4876698635939476, 1.3070103175531984, 1.1529797049092063, 2.314780636153873, 1.9234294306898572, 1.5697558727461935, 1.3193574716542222, 1.2524561316942173, 0.9556524419867579, 0.7985100096377913, 0.6718093801653432, 0.5751411733750736, 0.450054823019735, 0.38512487072844825, 0.3765932814328042 ], "peak_layer": 21, "peak_normalized": 2.314780636153873, "population_cv": 0.42845686821906764, "rest_mean": 1.8006899962428077e-05, "spike_contrast": 1.9157657573148825, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4497002343414354e-05, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.5729983715573326e-05, 1.6393925761803985e-05, 1.6268062609015033e-05, 1.5257418453984428e-05, 2.3866810806794092e-05, 2.4062108423095196e-05, 2.1139065211173147e-05, 1.9154053006786853e-05, 2.0778006728505716e-05, 1.9026971131097525e-05, 1.7595224562683143e-05, 1.643502946535591e-05, 1.5888830603216775e-05, 1.4854997971269768e-05, 1.4436292985919863e-05, 1.391248770232778e-05, 3.3414609788451344e-05, 3.0621420592069626e-05, 2.690281871764455e-05, 2.3732332920189947e-05, 4.764623736264184e-05, 3.9590868254890665e-05, 3.231103619327769e-05, 2.7156966098118573e-05, 2.577990380814299e-05, 1.9670651454362087e-05, 1.6436113583040424e-05, 1.3828173905494623e-05, 1.1838405953312758e-05, 9.26369375520153e-06, 7.927209480840247e-06, 7.751599696348421e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.388767562588328e-05, "first_to_last_ratio": 1.316792081076884, "imbalance_abs_log_ratio": 0.2751985371595874, "last_quartile_mean": 5.611187725662603e-05, "mean": 8.168265617314319e-05, "normalized": [ 0.7533431897136043, 0.7827542764436777, 0.7759396123840813, 0.7316445186865006, 1.1365162577009855, 1.1353975532420753, 1.004206629730762, 0.9167575048500237, 1.0078390566063014, 0.9220130731102486, 0.8550917608474935, 0.8013922543773514, 0.7721726808717908, 0.7274660564313035, 0.7065479131389877, 0.6779112969812525, 1.6398021444587536, 1.5049103837665865, 1.3267399301434915, 1.1597912607176666, 2.3220162536772593, 1.9302044249737353, 1.584049170842242, 1.3298950882483613, 1.2567261692224045, 0.9579394397382498, 0.801457903329196, 0.6752085554658249, 0.5811903488324587, 0.45264756329860717, 0.3896296913930569, 0.38079803677566654 ], "peak_layer": 21, "peak_normalized": 2.3220162536772593, "population_cv": 0.4315618932878077, "rest_mean": 7.132743997283763e-05, "spike_contrast": 1.9291428895694742, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001376008236547932, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.153507274575531e-05, 6.39374484308064e-05, 6.338080856949091e-05, 5.976266766083427e-05, 9.283366671297699e-05, 9.274228796130046e-05, 8.202626486308873e-05, 7.488318806281313e-05, 8.232297113863751e-05, 7.531247683800757e-05, 6.984616629779339e-05, 6.54598479741253e-05, 6.30731155979447e-05, 5.942135976511054e-05, 5.771271025878377e-05, 5.537359538720921e-05, 0.00013394339475780725, 0.00012292507744859904, 0.00010837164154509082, 9.473483078181744e-05, 0.0001896684552775696, 0.00015766422438900918, 0.0001293893437832594, 0.00010862936323974282, 0.00010265273158438504, 7.824703789083287e-05, 6.546521035488695e-05, 5.515282828127965e-05, 4.7473171434830874e-05, 3.69734552805312e-05, 3.1825988116906956e-05, 3.1104595109354705e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06552432896569371, "first_to_last_ratio": 1.3167920765276309, "imbalance_abs_log_ratio": 0.2751985337047875, "last_quartile_mean": 0.04976057354360819, "mean": 0.07243699650280178, "normalized": [ 0.7533431474468802, 0.7827543098589632, 0.7759395847698287, 0.731644488826528, 1.136516249702268, 1.1353975877134457, 1.0042065955447128, 0.9167574882575961, 1.007839058472007, 0.9220131219136145, 0.8550918118318299, 0.8013922821239102, 0.7721726890345729, 0.7274660147584042, 0.706547878986726, 0.6779113252026149, 1.6398021344750369, 1.5049103829869008, 1.3267399072401411, 1.1597912281377818, 2.3220164409485586, 1.9302043593714369, 1.584049135911302, 1.3298951182707694, 1.2567262311511898, 0.957939389851257, 0.8014578528316956, 0.6752085263417028, 0.581190365550621, 0.4526475557003154, 0.3896297084240016, 0.3807980283633875 ], "peak_layer": 21, "peak_normalized": 2.3220164409485586, "population_cv": 0.43156190791174726, "rest_mean": 0.0632538867217523, "spike_contrast": 1.9291429451165345, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1220257893204689, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.05456991493701935, 0.05670037120580673, 0.056206732988357544, 0.0529981292784214, 0.08232582360506058, 0.08224479109048843, 0.07274170964956284, 0.06640715897083282, 0.0730048343539238, 0.06678786128759384, 0.061940282583236694, 0.058050449937582016, 0.0559338703751564, 0.05269545316696167, 0.05118020623922348, 0.04910586029291153, 0.11878234148025513, 0.10901118814945221, 0.09610505402088165, 0.08401179313659668, 0.16819989681243896, 0.1398182064294815, 0.11474376171827316, 0.09633360803127289, 0.09103347361087799, 0.06939025223255157, 0.058055199682712555, 0.048910077661275864, 0.04209968447685242, 0.03278842940926552, 0.028223605826497078, 0.027583865448832512 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7910509743378498e-06, "first_to_last_ratio": 0.7327699142917456, "imbalance_abs_log_ratio": 0.3109235222896129, "last_quartile_mean": 3.8089049780865025e-06, "mean": 4.300465420215005e-06, "normalized": [ 0.6752230832161266, 0.6481028772772607, 0.634134920654564, 0.6489938741355444, 0.6688274895794406, 0.6540545586410917, 0.6334055531219255, 0.6293486411866674, 0.7505868142147136, 0.7168130527550195, 0.6943818949036571, 0.6987315584176189, 0.6285804128249975, 0.6319836168825069, 0.6466025849172643, 0.7005267172313684, 1.4911705866773426, 1.4376163460093667, 1.3858563653332445, 1.3109106922600342, 2.564962545843758, 2.293160284916099, 1.9861975918307744, 1.7842600166703042, 1.697776325534204, 1.2627187172796404, 1.0635667497903984, 0.874137655981081, 0.714444551315396, 0.5383938398368974, 0.4702297250968762, 0.4643003556648156 ], "peak_layer": 21, "peak_normalized": 2.564962545843758, "population_cv": 0.5485038394294482, "rest_mean": 3.4521019727064953e-06, "spike_contrast": 2.572817404289376, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.881628036760959e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.903773520301911e-06, 2.7871440124727087e-06, 2.7270752980257384e-06, 2.790975713651278e-06, 2.876269491025596e-06, 2.8127390123700025e-06, 2.723938678172999e-06, 2.7064920686825644e-06, 3.2278726393997204e-06, 3.0826297461317154e-06, 2.986165327456547e-06, 3.0048709049879108e-06, 2.7031883291783743e-06, 2.7178236905456288e-06, 2.7806920570583316e-06, 3.012590923390235e-06, 6.412727543647634e-06, 6.1824193835491315e-06, 5.9598273765004706e-06, 5.637526101054391e-06, 1.1030532732547726e-05, 9.861656508292072e-06, 8.541574061382562e-06, 7.673148502362892e-06, 7.301228379219538e-06, 5.430278179119341e-06, 4.573832029564073e-06, 3.7591987620544387e-06, 3.072444087592885e-06, 2.315344090675353e-06, 2.022206672336324e-06, 1.9967076241300674e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.460357715492137e-05, "first_to_last_ratio": 1.5303519803401489, "imbalance_abs_log_ratio": 0.4254977614562244, "last_quartile_mean": 3.568040415302676e-05, "mean": 5.699818092352871e-05, "normalized": [ 0.826499466074168, 0.8487872014190548, 0.8416021554467742, 0.8012967869639217, 1.1726116041287642, 1.1547121771623214, 1.0501726049153397, 0.9682225384122511, 1.0710773535839004, 0.9771144253342418, 0.8995261154993631, 0.8346162473012488, 0.8429729513138341, 0.7876515056477689, 0.7671474580293269, 0.735290511846575, 1.5890744132353023, 1.4640380695597026, 1.278360428063473, 1.1237055239693245, 2.371402173328689, 1.8882521909202439, 1.4825951619642506, 1.215335093029712, 1.183884081647654, 0.8905874439533303, 0.742080218171765, 0.6343188256622954, 0.482002642160446, 0.38702398334188143, 0.3477000388311781, 0.3403386090818978 ], "peak_layer": 21, "peak_normalized": 2.371402173328689, "population_cv": 0.42451190461357197, "rest_mean": 5.0366403094678254e-05, "spike_contrast": 1.8426922610466794, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.280978119932115e-05, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.710896610049531e-05, 4.837932647205889e-05, 4.7969791921786964e-05, 4.5672459236811846e-05, 6.683672836516052e-05, 6.581649358849972e-05, 5.9857928135897964e-05, 5.518692341865972e-05, 6.104946078266948e-05, 5.569374479819089e-05, 5.127135227667168e-05, 4.757160786539316e-05, 4.804792479262687e-05, 4.489470302360132e-05, 4.3726009607780725e-05, 4.191022162558511e-05, 9.057435090653598e-05, 8.344750676769763e-05, 7.286421896424145e-05, 6.404917075997218e-05, 0.0001351656101178378, 0.00010762694000732154, 8.45052272779867e-05, 6.927188951522112e-05, 6.74792390782386e-05, 5.0761864258674905e-05, 4.229722253512591e-05, 3.6155019188299775e-05, 2.747327380347997e-05, 2.205966302426532e-05, 1.9818269720417447e-05, 1.939868161571212e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.0483532302459935e-05, "first_to_last_ratio": 1.963155318608148, "imbalance_abs_log_ratio": 0.674553035245332, "last_quartile_mean": 2.062166549876565e-05, "mean": 3.870260115945712e-05, "normalized": [ 0.9359797302060725, 0.9576124055199563, 0.9480772177254478, 0.9102648966159671, 1.2467899109742078, 1.222393779568341, 1.1175155496691729, 1.0294929073580932, 1.1457881710656306, 1.0321570072444493, 0.9475098439722178, 0.8789722977603025, 0.9341127359239892, 0.8671918665148908, 0.8459313304801716, 0.8065696353523856, 1.568420236330532, 1.4485623429640861, 1.2469929473073715, 1.1052688868439036, 2.4248452552946893, 1.805023081283152, 1.2952633315442323, 1.0166745468576837, 1.067512212729438, 0.8179083685748635, 0.6887736887471813, 0.614681124858885, 0.32340289834365593, 0.26296753580630333, 0.24650039620284458, 0.2408438603598819 ], "peak_layer": 21, "peak_normalized": 2.4248452552946893, "population_cv": 0.43498346843587055, "rest_mean": 3.496232669998857e-05, "spike_contrast": 1.68467286934329, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.890008324058726e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.62248501915019e-05, 3.706209099618718e-05, 3.66930544259958e-05, 3.522961924318224e-05, 4.825401265406981e-05, 4.730981891043484e-05, 4.325075860833749e-05, 3.984405338997021e-05, 4.434498259797692e-05, 3.9947160985320807e-05, 3.667109558591619e-05, 3.401851427042857e-05, 3.6152592656435445e-05, 3.356258093845099e-05, 3.273974289186299e-05, 3.121634290437214e-05, 6.070194285712205e-05, 5.6063130614347756e-05, 4.826187068829313e-05, 4.277678090147674e-05, 9.384781878907233e-05, 6.985908839851618e-05, 5.0130060117226094e-05, 3.934794949600473e-05, 4.1315499402116984e-05, 3.165518137393519e-05, 2.6657333364710212e-05, 2.378975841565989e-05, 1.251653338840697e-05, 1.0177527656196617e-05, 9.540206519886851e-06, 9.321283869212493e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001419489935869933, "first_to_last_ratio": 1.1783352135158363, "imbalance_abs_log_ratio": 0.16410260630786871, "last_quartile_mean": 0.00012046571464452427, "mean": 0.00016912645673983207, "normalized": [ 0.6986475868430325, 0.7282565751265736, 0.7166008603062816, 0.6842111357537164, 1.0469692964748558, 1.026049824019131, 0.9391127634694222, 0.874606754649669, 1.0007023053561546, 0.9199914749419839, 0.8601715511639205, 0.8040475454029659, 0.7517874109118121, 0.7072705924944495, 0.6728256776786633, 0.6655104186910116, 1.6362823272721128, 1.4904042581473382, 1.3331379776154995, 1.1545215002514102, 2.429646858429421, 2.015441097306164, 1.7000198983358237, 1.4455290763603736, 1.310596018230462, 1.0039843631553074, 0.8374679195648416, 0.7043981788736101, 0.5858096739376918, 0.46318758062596044, 0.3998773352945733, 0.39293416331576775 ], "peak_layer": 21, "peak_normalized": 2.429646858429421, "population_cv": 0.4589394276046076, "rest_mean": 0.000144689356535242, "spike_contrast": 2.0809187700774854, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00030108679784461855, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.0001181597908725962, 0.00012316745414864272, 0.00012119616440031677, 0.00011571820505196229, 0.00017707020742818713, 0.00017353217117488384, 0.00015882881416473538, 0.00014791914145462215, 0.00016924523515626788, 0.00015559489838778973, 0.00014547776663675904, 0.00013598571240436286, 0.00012714714102912694, 0.0001196181692648679, 0.00011379262286936864, 0.00011255541903665289, 0.0002767386322375387, 0.0002520667912904173, 0.00022546890249941498, 0.00019526013056747615, 0.0004109175642952323, 0.0003408644115552306, 0.0002875183417927474, 0.0002444772107992321, 0.00022165646078065038, 0.00016980031796265393, 0.00014163798186928034, 0.0001191323681268841, 9.907591447699815e-05, 7.833727431716397e-05, 6.762983684893698e-05, 6.64555627736263e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.062413111329078674, 0.07552354782819748, 0.14653873443603516, 0.2105521410703659, 0.2186809629201889, 0.14272794127464294, 0.1287408471107483, 0.13253600895404816, 0.13371841609477997, 0.09585829079151154, 0.09883332997560501, 0.10020580142736435, 0.11237040907144547, 0.08538161963224411, 0.08663177490234375, 0.10358677804470062, 0.10874549299478531, 0.06467291712760925, 0.07278503477573395, 0.08346924930810928, 0.09857872873544693, 0.06093711405992508, 0.06834813207387924, 0.0680762529373169, 0.07369214296340942, 0.05633483827114105, 0.05850831791758537, 0.06479237228631973, 0.06705968081951141, 0.061747051775455475, 0.0654391348361969, 0.07205834239721298 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.513242389402876e-06, "first_to_last_ratio": 1.2607108267154772, "imbalance_abs_log_ratio": 0.2316757100785088, "last_quartile_mean": 7.545935346797705e-06, "mean": 1.3414615857243462e-05, "normalized": [ 0.54411236240655, 0.5345088335287688, 0.359970954850746, 0.5314334804876196, 0.3673058684703728, 0.903301495529033, 1.2731805653143804, 1.1595459992487693, 1.4831078521573031, 1.119758551480115, 0.913628335780701, 0.9147234218106974, 0.5358305370089425, 0.8430737978428791, 1.0000117212452202, 0.9024986222769764, 1.0159470806684705, 1.855308691450899, 1.5791792776893256, 1.8749806454984184, 1.9537650071631565, 2.5528608648736655, 2.0787080043386754, 1.2031304503621438, 1.0522901465907954, 0.940487300390985, 0.5556490720369505, 0.5996248916562782, 0.3826096482804215, 0.4666405513855537, 0.25941785990949606, 0.24340810826569018 ], "peak_layer": 22, "peak_normalized": 2.5528608648736655, "population_cv": 0.575130971839342, "rest_mean": 1.1506384529032706e-05, "spike_contrast": 2.0613829626268805, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3719065029581544e-05, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 7.299058324861107e-06, 7.1702306740917265e-06, 4.828872079087887e-06, 7.1289759944193065e-06, 4.927267127641244e-06, 1.21174425657955e-05, 1.7079228200600483e-05, 1.5554864148725756e-05, 1.989532211155165e-05, 1.502113082096912e-05, 1.2255973160790745e-05, 1.2270663319213782e-05, 7.187960818555439e-06, 1.1309511137369554e-05, 1.3414773093245458e-05, 1.2106672329537105e-05, 1.3628539818455465e-05, 2.4888253392418846e-05, 2.1184083379921503e-05, 2.5152145099127665e-05, 2.6209007046418265e-05, 3.424564783927053e-05, 2.7885069357580505e-05, 1.6139532817760482e-05, 1.4116068086877931e-05, 1.2616275853361003e-05, 7.453818852809491e-06, 8.043737580010202e-06, 5.1325614549568854e-06, 6.259803740249481e-06, 3.4799909371940885e-06, 3.265226268922561e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.6330423654362676e-05, "first_to_last_ratio": 1.2225177619555718, "imbalance_abs_log_ratio": 0.20091247146835384, "last_quartile_mean": 2.9717706183873815e-05, "mean": 5.264320969899927e-05, "normalized": [ 0.5035242395982877, 0.5277848791426298, 0.35262352810018, 0.5210210629397536, 0.3595263516776504, 0.8771435825340668, 1.2750208353143138, 1.1043598609817844, 1.421449261823989, 1.1627357712534139, 0.9147350742210957, 0.9225896301191776, 0.5357176288484936, 0.8409556403265339, 1.0036519032774602, 0.925597482954976, 1.0604518437864157, 1.8336332517284446, 1.6012013115443406, 1.8626503069202516, 1.929299073680775, 2.5897943935525722, 2.1504101537648252, 1.2080295776025163, 1.0642429477202255, 0.944975242275414, 0.5665571215143791, 0.570934005219875, 0.3855916338171456, 0.47604359079756364, 0.2627082854000429, 0.24504052756140657 ], "peak_layer": 22, "peak_normalized": 2.5897943935525722, "population_cv": 0.5809548582160158, "rest_mean": 4.4957737541322046e-05, "spike_contrast": 2.094072444457974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.414475935045629e-05, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 2.6507132133701816e-05, 2.778429006866645e-05, 1.8563234334578738e-05, 2.7428221073932946e-05, 1.8926621123682708e-05, 4.617565355147235e-05, 6.712118920404464e-05, 5.813704774482176e-05, 7.482965156668797e-05, 6.121014303062111e-05, 4.81545903312508e-05, 4.856807936448604e-05, 2.8201895474921912e-05, 4.427060412126593e-05, 5.2835457609035075e-05, 4.8726422392064705e-05, 5.58255887881387e-05, 9.652833978179842e-05, 8.429237641394138e-05, 9.805589070310816e-05, 0.00010156449570786208, 0.00013633508933708072, 0.00011320449266349897, 6.359455437632278e-05, 5.602516466751695e-05, 4.974652983946726e-05, 2.982538535434287e-05, 3.0055798561079428e-05, 2.0298781237215735e-05, 2.5060462576220743e-05, 1.3829807357979007e-05, 1.2899719877168536e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.032218182226642966, "first_to_last_ratio": 1.2225177500182758, "imbalance_abs_log_ratio": 0.20091246170383634, "last_quartile_mean": 0.026353958644904196, "mean": 0.046684524888405576, "normalized": [ 0.5035242503965048, 0.5277848868035033, 0.3526235212534414, 0.5210210441313687, 0.3595263305975145, 0.8771435091867963, 1.2750207710793637, 1.1043598828940213, 1.4214492710356383, 1.162735779950175, 0.9147351310550158, 0.9225896406648795, 0.5357176374824659, 0.8409556781269271, 1.003651856889631, 0.9255975130347041, 1.0604518407651282, 1.8336332774847264, 1.601201364484995, 1.8626502208000761, 1.9292991633227214, 2.589794548624206, 2.150410100335512, 1.2080294989428133, 1.0642429217970695, 0.9449752045153401, 0.5665571468541322, 0.5709340184911269, 0.38559161860900615, 0.4760435610967507, 0.2627082741971009, 0.2450405350973446 ], "peak_layer": 22, "peak_normalized": 2.589794548624206, "population_cv": 0.5809548711713294, "rest_mean": 0.03986897116044053, "spike_contrast": 2.094072472636495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08348851501941681, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.02350679039955139, 0.02463938668370247, 0.016462061554193497, 0.02432361990213394, 0.016784315928816795, 0.040949027985334396, 0.05952373892068863, 0.05155651643872261, 0.06635968387126923, 0.05428176745772362, 0.04270397499203682, 0.043070659041404724, 0.025009723380208015, 0.039259616285562515, 0.04685501009225845, 0.04321108013391495, 0.049506690353155136, 0.0856022983789444, 0.07475132495164871, 0.0869569405913353, 0.09006841480731964, 0.12090332806110382, 0.10039087384939194, 0.05639628320932388, 0.04968367516994476, 0.04411571845412254, 0.026449451223015785, 0.02665378339588642, 0.018001161515712738, 0.02222386747598648, 0.012264410965144634, 0.011439600959420204 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.2802849315439744e-06, "first_to_last_ratio": 0.4283455786552265, "imbalance_abs_log_ratio": 0.8478249824294246, "last_quartile_mean": 5.323470219309456e-06, "mean": 4.877433806171894e-06, "normalized": [ 1.4839433033627585, 0.703687558597217, 0.19787363001413222, 0.2183139267711944, 0.1618540975898947, 0.2679413581551488, 0.34335611131172533, 0.36316882023729685, 0.49344402674477783, 0.6027840686307443, 0.5647136700772823, 0.7110849136409753, 0.4430254500989309, 0.5976516238780146, 0.6050388629285002, 0.6350381439922589, 0.9181070377370916, 1.494677817366969, 1.5696877676314565, 1.6091270885003666, 1.7368727838973916, 3.0783329280416862, 2.262173099042413, 2.2065099626031874, 1.6586180705188411, 1.7272863652034096, 1.2173480016488631, 1.3531761425833577, 0.9843111324014115, 0.8080573379451803, 0.5264086923618475, 0.4563862064856749 ], "peak_layer": 22, "peak_normalized": 3.0783329280416862, "population_cv": 0.6993295019446811, "rest_mean": 3.8039455182149175e-06, "spike_contrast": 2.806105005980394, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0674270561139565e-05, "top_five_layers": [ 22, 23, 24, 21, 26 ], "top_five_spike_overlap": 4 }, "values": [ 7.237835234263912e-06, 3.4321894872846315e-06, 9.65115532380878e-07, 1.0648117267919588e-06, 7.894326472523971e-07, 1.306866238337534e-06, 1.6746967048675288e-06, 1.771331881172955e-06, 2.4067405774985673e-06, 2.940039394161431e-06, 2.7543535452423384e-06, 3.4682695968513144e-06, 2.160827307307045e-06, 2.9150062346161576e-06, 2.9510370040952694e-06, 3.097356511716498e-06, 4.478006303543225e-06, 7.290192115760874e-06, 7.656048182980157e-06, 7.84841085987864e-06, 8.471482033201028e-06, 1.5014365089882631e-05, 1.1033599548682105e-05, 1.0762106285255868e-05, 8.089799848676194e-06, 8.424724910582881e-06, 5.9375342971179634e-06, 6.600027063541347e-06, 4.800912392965984e-06, 3.941246177419089e-06, 2.5675235519884154e-06, 2.225993512183777e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.354378420932335e-05, "first_to_last_ratio": 1.2464041441561005, "imbalance_abs_log_ratio": 0.22026272102941338, "last_quartile_mean": 1.888936611749159e-05, "mean": 3.545669885340885e-05, "normalized": [ 0.4191906097330923, 0.49348811099284096, 0.38372804382004466, 0.5201360165053419, 0.3641940443046511, 0.8688909640601193, 1.2049617168759028, 1.0575314684579886, 1.2475942615588305, 1.1749760705863097, 0.940415402502436, 0.897650499398874, 0.5756566289364113, 0.8922773631416766, 1.048544229093483, 0.9399393226018595, 1.0859580556766202, 1.9234179989731328, 1.6483168700797706, 1.8071073699183404, 1.9702308124727184, 2.7499314904694696, 2.243433777969385, 1.2804717854341459, 1.0657725757024525, 0.9763545100314811, 0.5574845360072901, 0.4565738604950286, 0.3799339743020443, 0.40651690619449354, 0.21895640546763065, 0.2003643182361338 ], "peak_layer": 22, "peak_normalized": 2.7499314904694696, "population_cv": 0.6127980670514929, "rest_mean": 2.9796968680818315e-05, "spike_contrast": 2.215636177377244, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.601924178539775e-05, "top_five_layers": [ 22, 23, 21, 18, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.4863115211483091e-05, 1.7497459339210764e-05, 1.3605729691335e-05, 1.8442306100041606e-05, 1.2913118553115055e-05, 3.0808005249127746e-05, 4.272396472515538e-05, 3.7496574805118144e-05, 4.423557402333245e-05, 4.1660772694740444e-05, 3.334402572363615e-05, 3.182772343279794e-05, 2.0410883735166863e-05, 3.163720975862816e-05, 3.7177916965447366e-05, 3.3327145501971245e-05, 3.8504487747559324e-05, 6.819805275881663e-05, 5.844387487741187e-05, 6.40740618109703e-05, 6.985788058955222e-05, 9.750349272508174e-05, 7.954475586302578e-05, 4.5401302486425266e-05, 3.778877726290375e-05, 3.461830783635378e-05, 1.976656130864285e-05, 1.6188601875910535e-05, 1.3471204511006363e-05, 1.4413747521757614e-05, 7.763471330690663e-06, 7.104257292667171e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.6053550893957436e-05, "first_to_last_ratio": 1.291144701278337, "imbalance_abs_log_ratio": 0.2555291902297627, "last_quartile_mean": 1.2433579968273989e-05, "mean": 2.3879894939682345e-05, "normalized": [ 0.381944208893466, 0.5064170283475032, 0.4207326908745865, 0.5077824951609179, 0.39135241691859496, 0.887743098047013, 1.221771198045867, 1.0603545260365184, 1.112358047025732, 1.1817339073136295, 0.9153849269316471, 0.8293975374829873, 0.6253146359858851, 0.9134444345143908, 1.0884909424487381, 0.93703564258362, 1.0885287239694037, 2.0378246223538903, 1.6522638992705716, 1.6683493816940003, 1.9552451870072145, 2.8560629763771863, 2.298629545440282, 1.2964661784710367, 1.0642451847684595, 1.0310733143136432, 0.5500502286297949, 0.40889324961186074, 0.38796838095670966, 0.3474524989052327, 0.1966278305110652, 0.17906106110855208 ], "peak_layer": 22, "peak_normalized": 2.8560629763771863, "population_cv": 0.6286511089273724, "rest_mean": 1.9925871600679346e-05, "spike_contrast": 2.2699946018299357, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.523162097029854e-05, "top_five_layers": [ 22, 23, 18, 21, 20 ], "top_five_spike_overlap": 3 }, "values": [ 9.120787581196055e-06, 1.2093185432604514e-05, 1.0047052455774974e-05, 1.2125792636652477e-05, 9.34545460040681e-06, 2.1199211914790794e-05, 2.9175767849665135e-05, 2.5321154680568725e-05, 2.656299329828471e-05, 2.8219681553309783e-05, 2.1859295884496532e-05, 1.9805926058324985e-05, 1.4932447811588645e-05, 2.18129571294412e-05, 2.5993049348471686e-05, 2.2376312699634582e-05, 2.5993951567215845e-05, 4.8663037887308747e-05, 3.945588832721114e-05, 3.984000795753673e-05, 4.669104964705184e-05, 6.820248381700367e-05, 5.489103205036372e-05, 3.0959476134739816e-05, 2.5414063202333637e-05, 2.4621922420919873e-05, 1.3135141671227757e-05, 9.764327842276543e-06, 9.264644177164882e-06, 8.297129170387052e-06, 4.695451934821904e-06, 4.275959327060264e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.614753056055633e-05, "first_to_last_ratio": 1.1762933160121276, "imbalance_abs_log_ratio": 0.16236823675450227, "last_quartile_mean": 5.623387437481142e-05, "mean": 0.00010358891012174354, "normalized": [ 0.42097325737912505, 0.48924702737687037, 0.34042189089398245, 0.5266302780781597, 0.33345251919713514, 0.8199682417706958, 1.1522135672308953, 1.025557472234399, 1.3534387224479996, 1.1847086209851718, 0.9533429560557738, 0.955796607001976, 0.5321117819138612, 0.8503724980636721, 0.997244364596572, 0.928898529620989, 1.0923838319024162, 1.8584595513189417, 1.6516870083000426, 1.9684072477501107, 1.9630027958449041, 2.688216127574533, 2.254735452453152, 1.3158806474486227, 1.0888392323833342, 0.9550626820961028, 0.5732683646881589, 0.4722751280665548, 0.3639803606056138, 0.4532808789761308, 0.2263430485366378, 0.20979930720746648 ], "peak_layer": 22, "peak_normalized": 2.688216127574533, "population_cv": 0.6154722710504141, "rest_mean": 8.705046389810309e-05, "spike_contrast": 2.2159160456077167, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019289651972940192, "top_five_layers": [ 22, 23, 20, 21, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.3608160922303796e-05, 5.0680566346272826e-05, 3.526393265929073e-05, 5.4553056543227285e-05, 3.4541983040980995e-05, 8.493961649946868e-05, 0.00011935654765693471, 0.00010623638081597164, 0.00014020124217495322, 0.00012272267485968769, 9.875575779005885e-05, 9.900992881739512e-05, 5.5120879551395774e-05, 8.808916027192026e-05, 0.00010330345685360953, 9.622358629712835e-05, 0.0001131588505813852, 0.00019251579942647368, 0.00017109645705204457, 0.0002039051614701748, 0.00020334532018750906, 0.00027846937882713974, 0.0002335655881324783, 0.00013631064211949706, 0.00011279166938038543, 9.893390233628452e-05, 5.938424510532059e-05, 4.8922465794021264e-05, 3.770432886085473e-05, 4.6954872232163325e-05, 2.344662971154321e-05, 2.1732881577918306e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05931045487523079, 0.07934419065713882, 0.214645653963089, 0.09514089673757553, 0.12821291387081146, 0.1321839988231659, 0.12568901479244232, 0.1335759311914444, 0.07788944244384766, 0.08352623134851456, 0.07934392243623734, 0.09343641996383667, 0.07063088566064835, 0.06870647519826889, 0.07622843235731125, 0.08666495233774185, 0.057347409427165985, 0.05777324363589287, 0.06351501494646072, 0.07469023764133453, 0.051319729536771774, 0.05102822557091713, 0.05902284011244774, 0.06656526774168015, 0.050236333161592484, 0.05184732377529144, 0.061248794198036194, 0.06608261168003082, 0.048912134021520615, 0.062179163098335266, 0.07131517678499222, 0.06802479177713394 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.9132973458654305e-06, "first_to_last_ratio": 0.18177325947364456, "imbalance_abs_log_ratio": 1.7049951956609164, "last_quartile_mean": 2.702981373658986e-05, "mean": 1.9435898153119524e-05, "normalized": [ 0.5827233187547133, 0.21785343322460332, 0.09025019230653299, 0.27877726302073735, 0.20197289680560415, 0.11728555064621506, 0.16965705081784332, 0.36384007719584965, 0.6141661884449419, 0.4558215636950886, 0.47742147413318603, 0.4568805250530343, 0.5629610314949864, 0.5194178319866835, 0.4756935842462155, 0.4665970921430879, 1.370025243738621, 1.38943151782949, 1.4294802852403141, 1.357602686438849, 2.472173788023745, 2.540944975389688, 2.27148212683372, 1.9918126315455524, 2.108389723940819, 1.909399326521429, 1.5199007981214199, 1.3824424664562098, 1.4431666705716988, 1.096865155373051, 0.8791837858211463, 0.7863797441849225 ], "peak_layer": 22, "peak_normalized": 2.540944975389688, "population_cv": 0.7314263156880372, "rest_mean": 1.4839809797127532e-05, "spike_contrast": 2.982165936118834, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.425477527547628e-05, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1325751074764412e-05, 4.234177140460815e-06, 1.7540935459692264e-06, 5.418286491476465e-06, 3.925524652004242e-06, 2.2795500171923777e-06, 3.2974371606542263e-06, 7.07155868440168e-06, 1.1936871487705503e-05, 8.859301487973426e-06, 9.279115147364791e-06, 8.879883353074547e-06, 1.0941653272311669e-05, 1.0095352081407327e-05, 9.245532055501826e-06, 9.068733561434783e-06, 2.662767110450659e-05, 2.7004849471268244e-05, 2.778323323582299e-05, 2.638622754602693e-05, 4.804891796084121e-05, 4.9385547754354775e-05, 4.414829527377151e-05, 3.871266744681634e-05, 4.097844794159755e-05, 3.711089084390551e-05, 2.9540637115132995e-05, 2.6869010980590247e-05, 2.8049240427208133e-05, 2.1318559447536245e-05, 1.708772651909385e-05, 1.528399661765434e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.9200162171273405e-05, "first_to_last_ratio": 0.17782955919377144, "imbalance_abs_log_ratio": 1.7269297200365261, "last_quartile_mean": 0.00010796946389746154, "mean": 7.753401975207908e-05, "normalized": [ 0.5802604748565295, 0.21657891030507218, 0.08122798291669481, 0.27804637394409343, 0.19800644999728254, 0.11583331846598444, 0.16581158447272498, 0.34531750415538864, 0.6119033375078596, 0.4582687548340118, 0.4778600356941264, 0.4551699938366303, 0.5608630718384122, 0.5132558797569152, 0.4786206261633362, 0.46421909665654804, 1.3655188026701754, 1.3920864482886812, 1.4446997895948175, 1.3618673115230535, 2.481688366104822, 2.532895764609021, 2.2670762930167974, 2.0125790621760973, 2.1041793043599823, 1.9172636100383544, 1.5214473562286417, 1.3815731267036704, 1.4472156032763779, 1.1033023996856395, 0.8759464714083287, 0.7894168949139294 ], "peak_layer": 22, "peak_normalized": 2.532895764609021, "population_cv": 0.7341542937111465, "rest_mean": 5.916012609035264e-05, "spike_contrast": 2.9877056931125323, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017675304552540184, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.498992711887695e-05, 1.6792233509477228e-05, 6.297932031884557e-06, 2.1558053049375303e-05, 1.535223600512836e-05, 8.981022801890504e-06, 1.2856038665631786e-05, 2.6773854187922552e-05, 4.7443325456697494e-05, 3.5531418689060956e-05, 3.705040944623761e-05, 3.5291159292683005e-05, 4.34859684901312e-05, 3.9794791518943384e-05, 3.7109381082700565e-05, 3.59927726094611e-05, 0.00010587416181806475, 0.00010793405817821622, 0.00011201338202226907, 0.00010559104703133926, 0.00019241527479607612, 0.00019638559024315327, 0.00017577553808223456, 0.0001560433447593823, 0.0001631454797461629, 0.0001486531546106562, 0.00011796392936958, 0.00010711891809478402, 0.00011220844316994771, 8.554347004974261e-05, 6.791565101593733e-05, 6.120666512288153e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0170268957153894, "first_to_last_ratio": 0.17782955808409184, "imbalance_abs_log_ratio": 1.7269297262766563, "last_quartile_mean": 0.09574840031564236, "mean": 0.06875794463849161, "normalized": [ 0.5802604568764914, 0.21657890183963943, 0.08122798509807252, 0.27804637670503696, 0.19800643904545134, 0.115833321551524, 0.16581158131171497, 0.34531750831052266, 0.6119033445588002, 0.45826877711162134, 0.4778600229397992, 0.4551700187870312, 0.5608630663003571, 0.5132558897875106, 0.4786205986883147, 0.4642191256054569, 1.3655187793056378, 1.3920863761172964, 1.4446998184613569, 1.361867387227318, 2.481688342051939, 2.532895819110775, 2.267076382937339, 2.0125790037035514, 2.1041792403394113, 1.9172635512087937, 1.5214474269881526, 1.3815731149301655, 1.4472156023555418, 1.1033024068469601, 0.8759464966522335, 0.7894168372461831 ], "peak_layer": 22, "peak_normalized": 2.532895819110775, "population_cv": 0.7341542943028188, "rest_mean": 0.05246379186778709, "spike_contrast": 2.9877056922478897, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15674636960029603, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.03989751636981964, 0.014891520142555237, 0.005585069302469492, 0.019117897376418114, 0.013614515773952007, 0.007964461110532284, 0.011400863528251648, 0.023743322119116783, 0.042073216289281845, 0.03150961920619011, 0.03285667300224304, 0.03129655495285988, 0.038563791662454605, 0.035290420055389404, 0.03290896862745285, 0.03191875293850899, 0.09389026463031769, 0.09571699798107147, 0.09933459013700485, 0.09363920241594315, 0.17063578963279724, 0.17415671050548553, 0.15587951242923737, 0.13838079571723938, 0.14467903971672058, 0.13182710111141205, 0.1046115979552269, 0.09499412775039673, 0.09950757026672363, 0.07586080580949783, 0.06022828072309494, 0.05427867919206619 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.967008836042623e-07, "first_to_last_ratio": 0.13886164262645015, "imbalance_abs_log_ratio": 1.974277218356026, "last_quartile_mean": 7.177654424594948e-06, "mean": 4.984633447335796e-06, "normalized": [ 0.5560650351741472, 0.20102122926862334, 0.07002478683248901, 0.2844055838743635, 0.12143455655713824, 0.09092975294696704, 0.1066409068927962, 0.16911574529539783, 0.4501637856763599, 0.3830962086593998, 0.45012441999702146, 0.4287715723483553, 0.5612487151278214, 0.5091276890001657, 0.4770758153323667, 0.44669518124695695, 1.3364518589018204, 1.4488775934268328, 1.4466091632605513, 1.4235110430079823, 2.502127463836705, 2.5139466555424126, 2.39841810266287, 2.1044665924310153, 2.138548056523478, 2.0036229432372017, 1.6440023639406895, 1.4922570211503863, 1.464470142868618, 1.100253768757845, 0.86132188499381, 0.8151743612274127 ], "peak_layer": 22, "peak_normalized": 2.5139466555424126, "population_cv": 0.7762261395684433, "rest_mean": 3.7555508019640576e-06, "spike_contrast": 3.094534023149238, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1621679732343182e-05, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.7717803732230095e-06, 1.0020171430369373e-06, 3.4904789458778396e-07, 1.4176575859892182e-06, 6.053067522771016e-07, 4.532514878974325e-07, 5.315658313520544e-07, 8.429800004705612e-07, 2.243901462861686e-06, 1.9095941752311774e-06, 2.2437052393797785e-06, 2.1372691207943717e-06, 2.7976191177003784e-06, 2.5378149075550027e-06, 2.3780480660207104e-06, 2.226611741207307e-06, 6.661722636636114e-06, 7.2221237132907845e-06, 7.210816420410993e-06, 7.095680757629452e-06, 1.2472188245737925e-05, 1.253110258403467e-05, 1.1955235095229e-05, 1.0489994565432426e-05, 1.0659878171281889e-05, 9.987325938709546e-06, 8.194749170797877e-06, 7.438354259647895e-06, 7.2998468567675445e-06, 5.484361736307619e-06, 4.293373876862461e-06, 4.0633453863847535e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2451597399376624e-05, "first_to_last_ratio": 0.19401707978808486, "imbalance_abs_log_ratio": 1.6398090836493477, "last_quartile_mean": 6.417784152290551e-05, "mean": 4.8993523961371466e-05, "normalized": [ 0.6223930566187945, 0.21900570100977182, 0.07494974477635315, 0.2911875989654724, 0.19630689183835176, 0.10605512222211717, 0.17316480318369307, 0.3501196561590397, 0.648450442529022, 0.5057741493666605, 0.5380805526513028, 0.5110202884462354, 0.636128316167611, 0.591783883393767, 0.5520148879226475, 0.5333926201155943, 1.382390430732233, 1.3902405931289812, 1.434940184120226, 1.3725264922836518, 2.63138898922869, 2.6354613909169915, 2.2184615366462355, 1.9053627250852199, 2.10966594886892, 1.8441371017158037, 1.4431334012868662, 1.3129956485826366, 1.2970260777727707, 0.9656964298374909, 0.7844827226787779, 0.722262611748071 ], "peak_layer": 22, "peak_normalized": 2.6354613909169915, "population_cv": 0.7271026814282602, "rest_mean": 3.7198168683971744e-05, "spike_contrast": 3.029408340413438, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011268844245932996, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.0493229132844135e-05, 1.072986106009921e-05, 3.672052116598934e-06, 1.4266306607169099e-05, 9.617766409064643e-06, 5.196014171815477e-06, 8.48395393404644e-06, 1.715359576337505e-05, 3.1769872293807566e-05, 2.4779657906037755e-05, 2.6362462449469604e-05, 2.5036684746737592e-05, 3.116616790066473e-05, 2.8993577871005982e-05, 2.7045154638472013e-05, 2.613278411445208e-05, 6.772817869205028e-05, 6.811278581153601e-05, 7.030277629382908e-05, 6.724490958731622e-05, 0.00012892101949546486, 0.000129120540805161, 0.0001086902484530583, 9.335043432656676e-05, 0.0001033599692163989, 9.035077528096735e-05, 7.070419087540358e-05, 6.432828377000988e-05, 6.354587821988389e-05, 4.731287117465399e-05, 3.8434573070844635e-05, 3.5386190575081855e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.240943799364686e-06, "first_to_last_ratio": 0.24534451042071564, "imbalance_abs_log_ratio": 1.4050918911790493, "last_quartile_mean": 3.3589273243705975e-05, "mean": 2.9281088309573988e-05, "normalized": [ 0.697594278712878, 0.24149914140180637, 0.07261827844624033, 0.3161533325536177, 0.210190468546024, 0.1180112097175629, 0.20064229389807767, 0.3948312980233614, 0.7281804923556615, 0.5807663232829994, 0.6275700759618513, 0.5956218452297064, 0.7485593740948113, 0.7082998262177285, 0.6766344353399901, 0.6275549804018643, 1.4496279922272475, 1.4564659702931255, 1.4770283593746687, 1.4496530893722464, 2.8466255319856013, 2.810910306759418, 2.114703825619007, 1.6732013114486135, 2.0139259913891268, 1.6968132521341137, 1.3105709239526526, 1.2133397387383178, 1.0442280938677242, 0.7369046078385154, 0.6041734489015155, 0.5570999019139249 ], "peak_layer": 21, "peak_normalized": 2.8466255319856013, "population_cv": 0.7213237800230021, "rest_mean": 2.227600332473769e-05, "spike_contrast": 3.0125936977736663, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.710854722769e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.0426319679245353e-05, 7.071357686072588e-06, 2.1263422240735963e-06, 9.257313649868593e-06, 6.154605671326863e-06, 3.4554966532596154e-06, 5.87502472626511e-06, 1.1561090104805771e-05, 2.132191730197519e-05, 1.70054699992761e-05, 1.8375934814685024e-05, 1.7440455849282444e-05, 2.1918633137829602e-05, 2.0739789761137217e-05, 1.981259265448898e-05, 1.8375492800259963e-05, 4.244668525643647e-05, 4.2646908696042374e-05, 4.324899782659486e-05, 4.24474201281555e-05, 8.335229358635843e-05, 8.230651292251423e-05, 6.192082946654409e-05, 4.899315536022186e-05, 5.896994480281137e-05, 4.9684538680594414e-05, 3.8374942960217595e-05, 3.552790803951211e-05, 3.057613503187895e-05, 2.1577368897851557e-05, 1.7690856111585163e-05, 1.631249142519664e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.6764148262591334e-05, "first_to_last_ratio": 0.15860381719921426, "imbalance_abs_log_ratio": 1.8413459019801368, "last_quartile_mean": 0.00023179863455879968, "mean": 0.00016184205071567703, "normalized": [ 0.5514172677776774, 0.18824962050737556, 0.07412330605763978, 0.2681691846889825, 0.18477039778199453, 0.09341254286112102, 0.1477303241859572, 0.3094126957660134, 0.5998604868596944, 0.4469180103423267, 0.4751811779005959, 0.4510677352664099, 0.5417700474807862, 0.5021116806130881, 0.4526412355734704, 0.45370919271044824, 1.297304157819389, 1.297997666840438, 1.3953174020318846, 1.2918577787279846, 2.479178311227491, 2.5452011253115043, 2.3881988831304657, 2.1063821008782244, 2.2045217646011044, 2.008161694111016, 1.608368174721935, 1.4511166900922579, 1.4253132631711374, 1.0766448367874268, 0.8993268174418685, 0.7845644267322914 ], "peak_layer": 22, "peak_normalized": 2.5452011253115043, "population_cv": 0.7597506786457792, "rest_mean": 0.00012154048979735105, "spike_contrast": 3.122173362204994, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00037947047967463733, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 8.924250141717494e-05, 3.046670462936163e-05, 1.1996267858194187e-05, 4.3401050788816065e-05, 2.9903620088589378e-05, 1.5118077499209903e-05, 2.3908978619147092e-05, 5.007598520023748e-05, 9.708265133667737e-05, 7.233012729557231e-05, 7.690429629292339e-05, 7.30017272871919e-05, 8.768117550062016e-05, 8.126278407871723e-05, 7.325638580368832e-05, 7.342922617681324e-05, 0.00020995836530346423, 0.00021007060422562063, 0.000225821029744111, 0.00020907691214233637, 0.00040123530197888613, 0.0004119205696042627, 0.00038651100476272404, 0.00034090119879692793, 0.0003567843232303858, 0.000325005006743595, 0.0002603016037028283, 0.0002348517009522766, 0.00023067562142387033, 0.00017424640827812254, 0.0001455488963983953, 0.00012697551574092358 ] } } } } }, "learned": { "bits_per_byte": 1.8565786854442996, "forward": { "activation_sha256": { "attention_branch_output": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", "mlp_branch_output": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", "post_attention_state": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", "post_mlp_state": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", "pre_attention_input": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", "pre_mlp_input": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f" }, "logits_sha256": "a911233a29684ae3ced35122e9aee12a584fd2d6b7e0272c227ab94eb4e607ec", "loss_nats": 1.2868822813034058, "mixer_summary_sha256": "998a9ce6e3ee4cab0180aeb81cc12c18ccf6b1832b43c8f455133227a7e72a26" }, "loss_nats": 1.2868822813034058, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.21598131954669952, "group": 1, "latest_source_mass": 0.061274416744709015, "layer": 1, "max_source_mass": 0.9387255907058716, "mixer_index": 1, "normalized_entropy": 0.31159517863467795, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9387255907058716, "median": 0.9528161883354187, "p05": 0.8532108068466187, "p95": 0.9815421104431152 }, { "label": "current_group_partial", "mean": 0.061274416744709015, "median": 0.047183819115161896, "p05": 0.018457919359207153, "p95": 0.14678917825222015 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5377621054649353, "group": 1, "latest_source_mass": 0.2917928099632263, "layer": 2, "max_source_mass": 0.7082071900367737, "mixer_index": 2, "normalized_entropy": 0.7758267227322699, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.7082071900367737, "median": 0.7268136143684387, "p05": 0.38468095660209656, "p95": 0.923936128616333 }, { "label": "current_group_partial", "mean": 0.2917928099632263, "median": 0.27318641543388367, "p05": 0.07606390863656998, "p95": 0.6153190732002258 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.43774232268333435, "group": 1, "latest_source_mass": 0.170285165309906, "layer": 2, "max_source_mass": 0.829714834690094, "mixer_index": 3, "normalized_entropy": 0.6315286781224629, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.829714834690094, "median": 0.85542893409729, "p05": 0.687734842300415, "p95": 0.9260365962982178 }, { "label": "current_group_partial", "mean": 0.170285165309906, "median": 0.14457105100154877, "p05": 0.07396337389945984, "p95": 0.31226515769958496 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6749181151390076, "group": 1, "latest_source_mass": 0.45927006006240845, "layer": 3, "max_source_mass": 0.5407299995422363, "mixer_index": 4, "normalized_entropy": 0.9737010177171727, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5407299995422363, "median": 0.5310744643211365, "p05": 0.4232047200202942, "p95": 0.6916534900665283 }, { "label": "current_group_partial", "mean": 0.45927006006240845, "median": 0.4689255356788635, "p05": 0.3083464205265045, "p95": 0.576795220375061 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6663151383399963, "group": 1, "latest_source_mass": 0.49885618686676025, "layer": 3, "max_source_mass": 0.5011438131332397, "mixer_index": 5, "normalized_entropy": 0.9612895457523564, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5011438131332397, "median": 0.4804786741733551, "p05": 0.3554112911224365, "p95": 0.7207092642784119 }, { "label": "current_group_partial", "mean": 0.49885618686676025, "median": 0.5195213556289673, "p05": 0.27929070591926575, "p95": 0.6445887684822083 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6865031719207764, "group": 1, "latest_source_mass": 0.4691343307495117, "layer": 4, "max_source_mass": 0.5308656692504883, "mixer_index": 6, "normalized_entropy": 0.9904147216846476, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5308656692504883, "median": 0.5278033018112183, "p05": 0.45786383748054504, "p95": 0.6093498468399048 }, { "label": "current_group_partial", "mean": 0.4691343307495117, "median": 0.47219666838645935, "p05": 0.3906501829624176, "p95": 0.5421361327171326 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4133813679218292, "group": 1, "latest_source_mass": 0.15208598971366882, "layer": 4, "max_source_mass": 0.8479140400886536, "mixer_index": 7, "normalized_entropy": 0.5963832494967191, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.8479140400886536, "median": 0.8514594435691833, "p05": 0.7530128359794617, "p95": 0.9402707815170288 }, { "label": "current_group_partial", "mean": 0.15208598971366882, "median": 0.14854055643081665, "p05": 0.05972917377948761, "p95": 0.24698717892169952 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6850745677947998, "group": 2, "latest_source_mass": 0.4601486325263977, "layer": 5, "max_source_mass": 0.5398513674736023, "mixer_index": 8, "normalized_entropy": 0.9883536815967077, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5398513674736023, "median": 0.5387102365493774, "p05": 0.46027374267578125, "p95": 0.623892605304718 }, { "label": "completed_group_1", "mean": 0.4601486325263977, "median": 0.46128979325294495, "p05": 0.376107394695282, "p95": 0.5397262573242188 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0074119567871094, "group": 2, "latest_source_mass": 0.4627366364002228, "layer": 5, "max_source_mass": 0.4627366364002228, "mixer_index": 9, "normalized_entropy": 0.9169858804405273, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.32744842767715454, "median": 0.30657994747161865, "p05": 0.16577640175819397, "p95": 0.5453717708587646 }, { "label": "completed_group_1", "mean": 0.20981493592262268, "median": 0.2177819311618805, "p05": 0.14342713356018066, "p95": 0.2433384209871292 }, { "label": "current_group_partial", "mean": 0.4627366364002228, "median": 0.4847465753555298, "p05": 0.27109938859939575, "p95": 0.6081984639167786 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0489801168441772, "group": 2, "latest_source_mass": 0.4763939380645752, "layer": 6, "max_source_mass": 0.4763939380645752, "mixer_index": 10, "normalized_entropy": 0.9548228503031734, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28095996379852295, "median": 0.2765299677848816, "p05": 0.23549969494342804, "p95": 0.3424558639526367 }, { "label": "completed_group_1", "mean": 0.24264612793922424, "median": 0.23937124013900757, "p05": 0.21875721216201782, "p95": 0.2814398407936096 }, { "label": "current_group_partial", "mean": 0.4763939380645752, "median": 0.48882943391799927, "p05": 0.38478636741638184, "p95": 0.528400719165802 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.8527575731277466, "group": 2, "latest_source_mass": 0.6454645395278931, "layer": 6, "max_source_mass": 0.6454645395278931, "mixer_index": 11, "normalized_entropy": 0.7762133938639787, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20522089302539825, "median": 0.1591135561466217, "p05": 0.11052663624286652, "p95": 0.47355735301971436 }, { "label": "completed_group_1", "mean": 0.1493145078420639, "median": 0.15203402936458588, "p05": 0.10004506260156631, "p95": 0.18058395385742188 }, { "label": "current_group_partial", "mean": 0.6454645395278931, "median": 0.68653404712677, "p05": 0.4207671880722046, "p95": 0.7333818674087524 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.049942135810852, "group": 2, "latest_source_mass": 0.4395729899406433, "layer": 7, "max_source_mass": 0.4395729899406433, "mixer_index": 12, "normalized_entropy": 0.9556985177033998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35788866877555847, "median": 0.36233147978782654, "p05": 0.30507102608680725, "p95": 0.3985886573791504 }, { "label": "completed_group_1", "mean": 0.20253834128379822, "median": 0.19928529858589172, "p05": 0.180739626288414, "p95": 0.23214691877365112 }, { "label": "current_group_partial", "mean": 0.4395729899406433, "median": 0.43975552916526794, "p05": 0.3959323763847351, "p95": 0.4790603220462799 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9791408181190491, "group": 2, "latest_source_mass": 0.5214842557907104, "layer": 7, "max_source_mass": 0.5214842557907104, "mixer_index": 13, "normalized_entropy": 0.8912523810434521, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.31465935707092285, "median": 0.28992873430252075, "p05": 0.20534247159957886, "p95": 0.507034420967102 }, { "label": "completed_group_1", "mean": 0.1638563871383667, "median": 0.1667816936969757, "p05": 0.11925681680440903, "p95": 0.19515612721443176 }, { "label": "current_group_partial", "mean": 0.5214842557907104, "median": 0.5455081462860107, "p05": 0.35950881242752075, "p95": 0.6097541451454163 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0485806465148926, "group": 2, "latest_source_mass": 0.4457142949104309, "layer": 8, "max_source_mass": 0.4457142949104309, "mixer_index": 14, "normalized_entropy": 0.9544592367395849, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.346283882856369, "median": 0.353412926197052, "p05": 0.26436588168144226, "p95": 0.4084087312221527 }, { "label": "completed_group_1", "mean": 0.20800183713436127, "median": 0.20826736092567444, "p05": 0.18089595437049866, "p95": 0.23509056866168976 }, { "label": "current_group_partial", "mean": 0.4457142949104309, "median": 0.4416901469230652, "p05": 0.3883572816848755, "p95": 0.5136083364486694 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9801348447799683, "group": 2, "latest_source_mass": 0.543371319770813, "layer": 8, "max_source_mass": 0.543371319770813, "mixer_index": 15, "normalized_entropy": 0.8921571831025336, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2748759388923645, "median": 0.2577607035636902, "p05": 0.17991375923156738, "p95": 0.41774845123291016 }, { "label": "completed_group_1", "mean": 0.1817527413368225, "median": 0.17968356609344482, "p05": 0.130966454744339, "p95": 0.23766928911209106 }, { "label": "current_group_partial", "mean": 0.543371319770813, "median": 0.5535831451416016, "p05": 0.4452384412288666, "p95": 0.6142411828041077 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0688585042953491, "group": 3, "latest_source_mass": 0.36256325244903564, "layer": 9, "max_source_mass": 0.4051227569580078, "mixer_index": 16, "normalized_entropy": 0.9729169383233166, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4051227569580078, "median": 0.4008670449256897, "p05": 0.3286888897418976, "p95": 0.48389261960983276 }, { "label": "completed_group_1", "mean": 0.23231396079063416, "median": 0.23353952169418335, "p05": 0.2008797824382782, "p95": 0.2593720257282257 }, { "label": "completed_group_2", "mean": 0.36256325244903564, "median": 0.3638863265514374, "p05": 0.3081618845462799, "p95": 0.4163197875022888 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2915732860565186, "group": 3, "latest_source_mass": 0.36661261320114136, "layer": 9, "max_source_mass": 0.36661261320114136, "mixer_index": 17, "normalized_entropy": 0.931673187369201, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.26621371507644653, "median": 0.23494771122932434, "p05": 0.16815711557865143, "p95": 0.4694361090660095 }, { "label": "completed_group_1", "mean": 0.1108347550034523, "median": 0.11330658197402954, "p05": 0.06620892137289047, "p95": 0.13892118632793427 }, { "label": "completed_group_2", "mean": 0.2563389539718628, "median": 0.26728466153144836, "p05": 0.16010139882564545, "p95": 0.3091832995414734 }, { "label": "current_group_partial", "mean": 0.36661261320114136, "median": 0.36950358748435974, "p05": 0.29210716485977173, "p95": 0.42882072925567627 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3568191528320312, "group": 3, "latest_source_mass": 0.30908840894699097, "layer": 10, "max_source_mass": 0.30908840894699097, "mixer_index": 18, "normalized_entropy": 0.978738131586968, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.294538676738739, "median": 0.2961598038673401, "p05": 0.2467859834432602, "p95": 0.34424129128456116 }, { "label": "completed_group_1", "mean": 0.1710844486951828, "median": 0.17187124490737915, "p05": 0.1508811116218567, "p95": 0.1898101568222046 }, { "label": "completed_group_2", "mean": 0.2252884805202484, "median": 0.22875282168388367, "p05": 0.18443678319454193, "p95": 0.25181543827056885 }, { "label": "current_group_partial", "mean": 0.30908840894699097, "median": 0.3080518841743469, "p05": 0.2719307243824005, "p95": 0.3494151830673218 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.320021152496338, "group": 3, "latest_source_mass": 0.3312946557998657, "layer": 10, "max_source_mass": 0.3312946557998657, "mixer_index": 19, "normalized_entropy": 0.9521939852875004, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30465462803840637, "median": 0.311603307723999, "p05": 0.21955373883247375, "p95": 0.40013614296913147 }, { "label": "completed_group_1", "mean": 0.12911874055862427, "median": 0.12981033325195312, "p05": 0.09804825484752655, "p95": 0.15542283654212952 }, { "label": "completed_group_2", "mean": 0.23493197560310364, "median": 0.24103470146656036, "p05": 0.16810482740402222, "p95": 0.28502923250198364 }, { "label": "current_group_partial", "mean": 0.3312946557998657, "median": 0.325168639421463, "p05": 0.2647673785686493, "p95": 0.41786038875579834 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3578683137893677, "group": 3, "latest_source_mass": 0.30723780393600464, "layer": 11, "max_source_mass": 0.30723780393600464, "mixer_index": 20, "normalized_entropy": 0.9794949412420898, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29558902978897095, "median": 0.2938143014907837, "p05": 0.24335844814777374, "p95": 0.3542291522026062 }, { "label": "completed_group_1", "mean": 0.17812997102737427, "median": 0.17726755142211914, "p05": 0.15839283168315887, "p95": 0.20142368972301483 }, { "label": "completed_group_2", "mean": 0.21904319524765015, "median": 0.2218058705329895, "p05": 0.18076662719249725, "p95": 0.24698719382286072 }, { "label": "current_group_partial", "mean": 0.30723780393600464, "median": 0.30536961555480957, "p05": 0.26194074749946594, "p95": 0.35952210426330566 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.258887767791748, "group": 3, "latest_source_mass": 0.34303945302963257, "layer": 11, "max_source_mass": 0.35744374990463257, "mixer_index": 21, "normalized_entropy": 0.9080955698144659, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35744374990463257, "median": 0.359974205493927, "p05": 0.17910680174827576, "p95": 0.5207242965698242 }, { "label": "completed_group_1", "mean": 0.12067858874797821, "median": 0.11769672483205795, "p05": 0.08151324093341827, "p95": 0.16476252675056458 }, { "label": "completed_group_2", "mean": 0.17883820831775665, "median": 0.17648206651210785, "p05": 0.0995911955833435, "p95": 0.2739725708961487 }, { "label": "current_group_partial", "mean": 0.34303945302963257, "median": 0.33267199993133545, "p05": 0.22800606489181519, "p95": 0.48691368103027344 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3539760112762451, "group": 3, "latest_source_mass": 0.3175881803035736, "layer": 12, "max_source_mass": 0.3175881803035736, "mixer_index": 22, "normalized_entropy": 0.976687238475429, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2736307978630066, "median": 0.27999642491340637, "p05": 0.18969643115997314, "p95": 0.3464604914188385 }, { "label": "completed_group_1", "mean": 0.1722733974456787, "median": 0.17083553969860077, "p05": 0.1506597101688385, "p95": 0.1988438367843628 }, { "label": "completed_group_2", "mean": 0.2365075945854187, "median": 0.24049004912376404, "p05": 0.19099700450897217, "p95": 0.26942259073257446 }, { "label": "current_group_partial", "mean": 0.3175881803035736, "median": 0.3092706799507141, "p05": 0.268545538187027, "p95": 0.3924510180950165 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3203014135360718, "group": 3, "latest_source_mass": 0.3395293354988098, "layer": 12, "max_source_mass": 0.3395293354988098, "mixer_index": 23, "normalized_entropy": 0.9523961508935896, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28207293152809143, "median": 0.29242122173309326, "p05": 0.1354818046092987, "p95": 0.4195851981639862 }, { "label": "completed_group_1", "mean": 0.15420743823051453, "median": 0.15396761894226074, "p05": 0.11267110705375671, "p95": 0.1951446533203125 }, { "label": "completed_group_2", "mean": 0.22419027984142303, "median": 0.22542357444763184, "p05": 0.1655150204896927, "p95": 0.2747117578983307 }, { "label": "current_group_partial", "mean": 0.3395293354988098, "median": 0.32681703567504883, "p05": 0.2524653375148773, "p95": 0.46652013063430786 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3734662532806396, "group": 4, "latest_source_mass": 0.283242404460907, "layer": 13, "max_source_mass": 0.283242404460907, "mixer_index": 24, "normalized_entropy": 0.9907464762181619, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2766023874282837, "median": 0.282479852437973, "p05": 0.22528076171875, "p95": 0.3097468614578247 }, { "label": "completed_group_1", "mean": 0.21176087856292725, "median": 0.2058793306350708, "p05": 0.18811854720115662, "p95": 0.2617422640323639 }, { "label": "completed_group_2", "mean": 0.22839435935020447, "median": 0.22951346635818481, "p05": 0.20004916191101074, "p95": 0.2551718056201935 }, { "label": "completed_group_3", "mean": 0.283242404460907, "median": 0.2849352955818176, "p05": 0.2205706238746643, "p95": 0.34444743394851685 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5325813293457031, "group": 4, "latest_source_mass": 0.27591708302497864, "layer": 13, "max_source_mass": 0.27591708302497864, "mixer_index": 25, "normalized_entropy": 0.9522463199762954, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17451074719429016, "median": 0.16826045513153076, "p05": 0.08149556815624237, "p95": 0.3126368522644043 }, { "label": "completed_group_1", "mean": 0.10643941164016724, "median": 0.10269765555858612, "p05": 0.08178127557039261, "p95": 0.13947664201259613 }, { "label": "completed_group_2", "mean": 0.1691833734512329, "median": 0.17118802666664124, "p05": 0.12382284551858902, "p95": 0.2069271057844162 }, { "label": "completed_group_3", "mean": 0.27394935488700867, "median": 0.27278125286102295, "p05": 0.2122008204460144, "p95": 0.33945947885513306 }, { "label": "current_group_partial", "mean": 0.27591708302497864, "median": 0.27714359760284424, "p05": 0.21523593366146088, "p95": 0.33170604705810547 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.560360074043274, "group": 4, "latest_source_mass": 0.3255118429660797, "layer": 14, "max_source_mass": 0.3255118429660797, "mixer_index": 26, "normalized_entropy": 0.9695062244951087, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19377154111862183, "median": 0.19472965598106384, "p05": 0.17005598545074463, "p95": 0.22297024726867676 }, { "label": "completed_group_1", "mean": 0.17211595177650452, "median": 0.17136064171791077, "p05": 0.1543424427509308, "p95": 0.1928609162569046 }, { "label": "completed_group_2", "mean": 0.14605136215686798, "median": 0.14629822969436646, "p05": 0.13246256113052368, "p95": 0.15914371609687805 }, { "label": "completed_group_3", "mean": 0.16254930198192596, "median": 0.16272446513175964, "p05": 0.14075018465518951, "p95": 0.18463709950447083 }, { "label": "current_group_partial", "mean": 0.3255118429660797, "median": 0.3261105418205261, "p05": 0.3035835921764374, "p95": 0.3457673192024231 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.540909767150879, "group": 4, "latest_source_mass": 0.2995486855506897, "layer": 14, "max_source_mass": 0.2995486855506897, "mixer_index": 27, "normalized_entropy": 0.9574210693349581, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21921303868293762, "median": 0.21722173690795898, "p05": 0.1413341462612152, "p95": 0.2906968593597412 }, { "label": "completed_group_1", "mean": 0.10205383598804474, "median": 0.09984464198350906, "p05": 0.07455356419086456, "p95": 0.13362759351730347 }, { "label": "completed_group_2", "mean": 0.16381604969501495, "median": 0.16600549221038818, "p05": 0.11865930259227753, "p95": 0.2019941210746765 }, { "label": "completed_group_3", "mean": 0.215368390083313, "median": 0.2136734575033188, "p05": 0.1728287935256958, "p95": 0.2628125846385956 }, { "label": "current_group_partial", "mean": 0.2995486855506897, "median": 0.2982075810432434, "p05": 0.23863884806632996, "p95": 0.3662579655647278 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5142419338226318, "group": 4, "latest_source_mass": 0.369165301322937, "layer": 15, "max_source_mass": 0.369165301322937, "mixer_index": 28, "normalized_entropy": 0.940851412859105, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2000826597213745, "median": 0.20369741320610046, "p05": 0.15802279114723206, "p95": 0.24072687327861786 }, { "label": "completed_group_1", "mean": 0.15480534732341766, "median": 0.15228059887886047, "p05": 0.13543802499771118, "p95": 0.1815909892320633 }, { "label": "completed_group_2", "mean": 0.10827909409999847, "median": 0.10761409997940063, "p05": 0.09670238941907883, "p95": 0.12229038774967194 }, { "label": "completed_group_3", "mean": 0.16766759753227234, "median": 0.16637766361236572, "p05": 0.1398160457611084, "p95": 0.20042110979557037 }, { "label": "current_group_partial", "mean": 0.369165301322937, "median": 0.3697334825992584, "p05": 0.3279486298561096, "p95": 0.40613043308258057 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5333527326583862, "group": 4, "latest_source_mass": 0.29974091053009033, "layer": 15, "max_source_mass": 0.29974091053009033, "mixer_index": 29, "normalized_entropy": 0.9527256198031004, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21770668029785156, "median": 0.19699469208717346, "p05": 0.1311802864074707, "p95": 0.3720983564853668 }, { "label": "completed_group_1", "mean": 0.10525081306695938, "median": 0.10493603348731995, "p05": 0.08330865204334259, "p95": 0.12839283049106598 }, { "label": "completed_group_2", "mean": 0.15970060229301453, "median": 0.16236470639705658, "p05": 0.11309029906988144, "p95": 0.19861777126789093 }, { "label": "completed_group_3", "mean": 0.217601016163826, "median": 0.21904586255550385, "p05": 0.16268108785152435, "p95": 0.2699928283691406 }, { "label": "current_group_partial", "mean": 0.29974091053009033, "median": 0.30079299211502075, "p05": 0.22156940400600433, "p95": 0.37281978130340576 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.512608289718628, "group": 4, "latest_source_mass": 0.35851576924324036, "layer": 16, "max_source_mass": 0.35851576924324036, "mixer_index": 30, "normalized_entropy": 0.9398363727066501, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21392285823822021, "median": 0.22146853804588318, "p05": 0.1518818885087967, "p95": 0.25874194502830505 }, { "label": "completed_group_1", "mean": 0.1860550194978714, "median": 0.1838144212961197, "p05": 0.1660098135471344, "p95": 0.21311265230178833 }, { "label": "completed_group_2", "mean": 0.10291537642478943, "median": 0.1024642288684845, "p05": 0.09207107126712799, "p95": 0.11504144221544266 }, { "label": "completed_group_3", "mean": 0.1385909616947174, "median": 0.13646918535232544, "p05": 0.11220520734786987, "p95": 0.1728362739086151 }, { "label": "current_group_partial", "mean": 0.35851576924324036, "median": 0.3558155298233032, "p05": 0.31809645891189575, "p95": 0.40622422099113464 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5262645483016968, "group": 4, "latest_source_mass": 0.3142552077770233, "layer": 16, "max_source_mass": 0.3142552077770233, "mixer_index": 31, "normalized_entropy": 0.9483214832396903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20246261358261108, "median": 0.18459102511405945, "p05": 0.12276223301887512, "p95": 0.38892465829849243 }, { "label": "completed_group_1", "mean": 0.10530521720647812, "median": 0.10532709956169128, "p05": 0.0785030871629715, "p95": 0.1342671513557434 }, { "label": "completed_group_2", "mean": 0.16650916635990143, "median": 0.17111444473266602, "p05": 0.1085265576839447, "p95": 0.2094469517469406 }, { "label": "completed_group_3", "mean": 0.21146778762340546, "median": 0.2099919468164444, "p05": 0.15801574289798737, "p95": 0.26882117986679077 }, { "label": "current_group_partial", "mean": 0.3142552077770233, "median": 0.31609731912612915, "p05": 0.22627028822898865, "p95": 0.397154837846756 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5794764757156372, "group": 5, "latest_source_mass": 0.2760102152824402, "layer": 17, "max_source_mass": 0.2760102152824402, "mixer_index": 32, "normalized_entropy": 0.9813839126772218, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2210189402103424, "median": 0.22435861825942993, "p05": 0.17291823029518127, "p95": 0.262204647064209 }, { "label": "completed_group_1", "mean": 0.18839938938617706, "median": 0.18652752041816711, "p05": 0.16997642815113068, "p95": 0.2126670628786087 }, { "label": "completed_group_2", "mean": 0.14016608893871307, "median": 0.14067986607551575, "p05": 0.1265786588191986, "p95": 0.15091627836227417 }, { "label": "completed_group_3", "mean": 0.17440536618232727, "median": 0.17192018032073975, "p05": 0.15062645077705383, "p95": 0.20691406726837158 }, { "label": "completed_group_4", "mean": 0.2760102152824402, "median": 0.27511972188949585, "p05": 0.24412119388580322, "p95": 0.30901142954826355 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6602650880813599, "group": 5, "latest_source_mass": 0.3575916588306427, "layer": 17, "max_source_mass": 0.3575916588306427, "mixer_index": 33, "normalized_entropy": 0.9266115885502495, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12695768475532532, "median": 0.11564654856920242, "p05": 0.07159632444381714, "p95": 0.21949690580368042 }, { "label": "completed_group_1", "mean": 0.08036886155605316, "median": 0.07964898645877838, "p05": 0.06650256365537643, "p95": 0.09638062119483948 }, { "label": "completed_group_2", "mean": 0.12672865390777588, "median": 0.1287270039319992, "p05": 0.09939455986022949, "p95": 0.14712141454219818 }, { "label": "completed_group_3", "mean": 0.15391482412815094, "median": 0.15372109413146973, "p05": 0.13333335518836975, "p95": 0.17581847310066223 }, { "label": "completed_group_4", "mean": 0.154438316822052, "median": 0.15402798354625702, "p05": 0.12481103092432022, "p95": 0.18498417735099792 }, { "label": "current_group_partial", "mean": 0.3575916588306427, "median": 0.36001941561698914, "p05": 0.3110165596008301, "p95": 0.39699214696884155 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.5925545692443848, "group": 5, "latest_source_mass": 0.41908538341522217, "layer": 18, "max_source_mass": 0.41908538341522217, "mixer_index": 34, "normalized_entropy": 0.8888216284580353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14075040817260742, "median": 0.1455841362476349, "p05": 0.09744156897068024, "p95": 0.1818566620349884 }, { "label": "completed_group_1", "mean": 0.11028306186199188, "median": 0.10887216031551361, "p05": 0.09629634767770767, "p95": 0.12874113023281097 }, { "label": "completed_group_2", "mean": 0.07537610828876495, "median": 0.07517752051353455, "p05": 0.0666593387722969, "p95": 0.08491044491529465 }, { "label": "completed_group_3", "mean": 0.10244421660900116, "median": 0.10141433775424957, "p05": 0.08653581142425537, "p95": 0.12175601720809937 }, { "label": "completed_group_4", "mean": 0.1520608365535736, "median": 0.1513972282409668, "p05": 0.1301831305027008, "p95": 0.17589205503463745 }, { "label": "current_group_partial", "mean": 0.41908538341522217, "median": 0.4191858768463135, "p05": 0.3649080991744995, "p95": 0.4721684455871582 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5918848514556885, "group": 5, "latest_source_mass": 0.4151553511619568, "layer": 18, "max_source_mass": 0.4151553511619568, "mixer_index": 35, "normalized_entropy": 0.8884478518433735, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11773698031902313, "median": 0.10857899487018585, "p05": 0.06324320286512375, "p95": 0.22339528799057007 }, { "label": "completed_group_1", "mean": 0.07344591617584229, "median": 0.07228962332010269, "p05": 0.05982384458184242, "p95": 0.09060496836900711 }, { "label": "completed_group_2", "mean": 0.11976176500320435, "median": 0.12151201069355011, "p05": 0.08953659236431122, "p95": 0.14441059529781342 }, { "label": "completed_group_3", "mean": 0.14735516905784607, "median": 0.14620742201805115, "p05": 0.11767779290676117, "p95": 0.17902395129203796 }, { "label": "completed_group_4", "mean": 0.126544788479805, "median": 0.12564796209335327, "p05": 0.09858805686235428, "p95": 0.1567486822605133 }, { "label": "current_group_partial", "mean": 0.4151553511619568, "median": 0.4184337854385376, "p05": 0.3375611901283264, "p95": 0.4846380352973938 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6208878755569458, "group": 5, "latest_source_mass": 0.3991435170173645, "layer": 19, "max_source_mass": 0.3991435170173645, "mixer_index": 36, "normalized_entropy": 0.9046347477964072, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14785674214363098, "median": 0.15170978009700775, "p05": 0.09146977216005325, "p95": 0.20172694325447083 }, { "label": "completed_group_1", "mean": 0.12492769956588745, "median": 0.12345598638057709, "p05": 0.10790181905031204, "p95": 0.14623555541038513 }, { "label": "completed_group_2", "mean": 0.08781818300485611, "median": 0.08839264512062073, "p05": 0.07453933358192444, "p95": 0.09894395619630814 }, { "label": "completed_group_3", "mean": 0.10019385814666748, "median": 0.09901996701955795, "p05": 0.08430047333240509, "p95": 0.12013883143663406 }, { "label": "completed_group_4", "mean": 0.14006000757217407, "median": 0.13918349146842957, "p05": 0.11816112697124481, "p95": 0.16475799679756165 }, { "label": "current_group_partial", "mean": 0.3991435170173645, "median": 0.39726996421813965, "p05": 0.34116071462631226, "p95": 0.4634247422218323 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5470075607299805, "group": 5, "latest_source_mass": 0.44330859184265137, "layer": 19, "max_source_mass": 0.44330859184265137, "mixer_index": 37, "normalized_entropy": 0.8634013589985261, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12043030560016632, "median": 0.1012125313282013, "p05": 0.061379045248031616, "p95": 0.25343188643455505 }, { "label": "completed_group_1", "mean": 0.06855648756027222, "median": 0.06761305779218674, "p05": 0.05337192490696907, "p95": 0.08790726959705353 }, { "label": "completed_group_2", "mean": 0.11753211170434952, "median": 0.1186976358294487, "p05": 0.08412563800811768, "p95": 0.14895018935203552 }, { "label": "completed_group_3", "mean": 0.1339646875858307, "median": 0.13253431022167206, "p05": 0.09857159107923508, "p95": 0.1730121374130249 }, { "label": "completed_group_4", "mean": 0.1162077933549881, "median": 0.11592038720846176, "p05": 0.08609852194786072, "p95": 0.14767511188983917 }, { "label": "current_group_partial", "mean": 0.44330859184265137, "median": 0.4434061646461487, "p05": 0.3389148414134979, "p95": 0.5490255355834961 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.683492660522461, "group": 5, "latest_source_mass": 0.3365415334701538, "layer": 20, "max_source_mass": 0.3365415334701538, "mixer_index": 38, "normalized_entropy": 0.9395751435586169, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1658952683210373, "median": 0.1726911962032318, "p05": 0.10183650255203247, "p95": 0.22522984445095062 }, { "label": "completed_group_1", "mean": 0.1435280442237854, "median": 0.14210543036460876, "p05": 0.12450659275054932, "p95": 0.16782894730567932 }, { "label": "completed_group_2", "mean": 0.09461991488933563, "median": 0.0948714017868042, "p05": 0.08168491721153259, "p95": 0.10723742842674255 }, { "label": "completed_group_3", "mean": 0.10784181207418442, "median": 0.10693192481994629, "p05": 0.08930590003728867, "p95": 0.12958656251430511 }, { "label": "completed_group_4", "mean": 0.15157344937324524, "median": 0.15053030848503113, "p05": 0.128616601228714, "p95": 0.17799487709999084 }, { "label": "current_group_partial", "mean": 0.3365415334701538, "median": 0.33349889516830444, "p05": 0.2778944969177246, "p95": 0.40724658966064453 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5540567636489868, "group": 5, "latest_source_mass": 0.42266085743904114, "layer": 20, "max_source_mass": 0.42266085743904114, "mixer_index": 39, "normalized_entropy": 0.8673355940563396, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12987135350704193, "median": 0.1083056628704071, "p05": 0.03783264383673668, "p95": 0.3301126956939697 }, { "label": "completed_group_1", "mean": 0.07178181409835815, "median": 0.07081792503595352, "p05": 0.05334944650530815, "p95": 0.09311570972204208 }, { "label": "completed_group_2", "mean": 0.12132491171360016, "median": 0.12282539904117584, "p05": 0.07926258444786072, "p95": 0.15763431787490845 }, { "label": "completed_group_3", "mean": 0.1441602110862732, "median": 0.14273977279663086, "p05": 0.09847753494977951, "p95": 0.19304539263248444 }, { "label": "completed_group_4", "mean": 0.11020088195800781, "median": 0.10934801399707794, "p05": 0.07139573246240616, "p95": 0.1517774760723114 }, { "label": "current_group_partial", "mean": 0.42266085743904114, "median": 0.42266106605529785, "p05": 0.28481048345565796, "p95": 0.5570361018180847 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7208337783813477, "group": 6, "latest_source_mass": 0.27070939540863037, "layer": 21, "max_source_mass": 0.27070939540863037, "mixer_index": 40, "normalized_entropy": 0.9604156182429642, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.18048501014709473, "median": 0.18919861316680908, "p05": 0.11199048161506653, "p95": 0.23807701468467712 }, { "label": "completed_group_1", "mean": 0.1685236394405365, "median": 0.1669977903366089, "p05": 0.14678362011909485, "p95": 0.19381441175937653 }, { "label": "completed_group_2", "mean": 0.08726631104946136, "median": 0.08729851245880127, "p05": 0.07629868388175964, "p95": 0.0980045348405838 }, { "label": "completed_group_3", "mean": 0.11160479485988617, "median": 0.1099197044968605, "p05": 0.09190923720598221, "p95": 0.13723208010196686 }, { "label": "completed_group_4", "mean": 0.18141084909439087, "median": 0.18020854890346527, "p05": 0.15656134486198425, "p95": 0.2108723670244217 }, { "label": "completed_group_5", "mean": 0.27070939540863037, "median": 0.2684924602508545, "p05": 0.2159801721572876, "p95": 0.33192309737205505 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6729483604431152, "group": 6, "latest_source_mass": 0.3803577423095703, "layer": 21, "max_source_mass": 0.3803577423095703, "mixer_index": 41, "normalized_entropy": 0.8597253893019091, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09229261428117752, "median": 0.07798201590776443, "p05": 0.04784023016691208, "p95": 0.20285779237747192 }, { "label": "completed_group_1", "mean": 0.051420148462057114, "median": 0.05041736364364624, "p05": 0.040625106543302536, "p95": 0.06547587364912033 }, { "label": "completed_group_2", "mean": 0.08079522103071213, "median": 0.08136115223169327, "p05": 0.05947277322411537, "p95": 0.10017222166061401 }, { "label": "completed_group_3", "mean": 0.09550285339355469, "median": 0.09445887804031372, "p05": 0.07334083318710327, "p95": 0.12144046276807785 }, { "label": "completed_group_4", "mean": 0.07590493559837341, "median": 0.07492390275001526, "p05": 0.055510398000478745, "p95": 0.10084333270788193 }, { "label": "completed_group_5", "mean": 0.22372648119926453, "median": 0.22069036960601807, "p05": 0.1407623589038849, "p95": 0.32290777564048767 }, { "label": "current_group_partial", "mean": 0.3803577423095703, "median": 0.37903881072998047, "p05": 0.2855539321899414, "p95": 0.476589173078537 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.710235595703125, "group": 6, "latest_source_mass": 0.4011716842651367, "layer": 22, "max_source_mass": 0.4011716842651367, "mixer_index": 42, "normalized_entropy": 0.8788872376935791, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11657392978668213, "median": 0.11877645552158356, "p05": 0.06819256395101547, "p95": 0.1684979349374771 }, { "label": "completed_group_1", "mean": 0.10617930442094803, "median": 0.10509790480136871, "p05": 0.090352863073349, "p95": 0.12587188184261322 }, { "label": "completed_group_2", "mean": 0.061112720519304276, "median": 0.061012350022792816, "p05": 0.05164862796664238, "p95": 0.07133162766695023 }, { "label": "completed_group_3", "mean": 0.06796726584434509, "median": 0.06712733209133148, "p05": 0.05444999784231186, "p95": 0.08481802046298981 }, { "label": "completed_group_4", "mean": 0.103337861597538, "median": 0.10192783921957016, "p05": 0.0832459107041359, "p95": 0.12805812060832977 }, { "label": "completed_group_5", "mean": 0.14365722239017487, "median": 0.14159655570983887, "p05": 0.10498540848493576, "p95": 0.19020715355873108 }, { "label": "current_group_partial", "mean": 0.4011716842651367, "median": 0.40084803104400635, "p05": 0.32083970308303833, "p95": 0.47994357347488403 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5640671253204346, "group": 6, "latest_source_mass": 0.4689176678657532, "layer": 22, "max_source_mass": 0.4689176678657532, "mixer_index": 43, "normalized_entropy": 0.8037715030571925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.090422622859478, "median": 0.0774901732802391, "p05": 0.04492149129509926, "p95": 0.1988787055015564 }, { "label": "completed_group_1", "mean": 0.04561382159590721, "median": 0.044634923338890076, "p05": 0.03501313924789429, "p95": 0.05900419130921364 }, { "label": "completed_group_2", "mean": 0.07565715163946152, "median": 0.07550699263811111, "p05": 0.053458306938409805, "p95": 0.09821777045726776 }, { "label": "completed_group_3", "mean": 0.08878682553768158, "median": 0.08692863583564758, "p05": 0.06425619125366211, "p95": 0.11807316541671753 }, { "label": "completed_group_4", "mean": 0.06555216014385223, "median": 0.06414324045181274, "p05": 0.04460844025015831, "p95": 0.09090004861354828 }, { "label": "completed_group_5", "mean": 0.165049746632576, "median": 0.15503892302513123, "p05": 0.08368627727031708, "p95": 0.2792147397994995 }, { "label": "current_group_partial", "mean": 0.4689176678657532, "median": 0.46879667043685913, "p05": 0.3151778280735016, "p95": 0.6170428991317749 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7556228637695312, "group": 6, "latest_source_mass": 0.3706088662147522, "layer": 23, "max_source_mass": 0.3706088662147522, "mixer_index": 44, "normalized_entropy": 0.9022116795175967, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13281464576721191, "median": 0.13658174872398376, "p05": 0.08060654997825623, "p95": 0.17402096092700958 }, { "label": "completed_group_1", "mean": 0.12175846099853516, "median": 0.12066285312175751, "p05": 0.10334118455648422, "p95": 0.14384035766124725 }, { "label": "completed_group_2", "mean": 0.07235272228717804, "median": 0.07218228280544281, "p05": 0.059715576469898224, "p95": 0.08584222197532654 }, { "label": "completed_group_3", "mean": 0.0707377940416336, "median": 0.06942892074584961, "p05": 0.05500924959778786, "p95": 0.09052760899066925 }, { "label": "completed_group_4", "mean": 0.10551950335502625, "median": 0.103996641933918, "p05": 0.08408617228269577, "p95": 0.13205930590629578 }, { "label": "completed_group_5", "mean": 0.12620802223682404, "median": 0.12379512935876846, "p05": 0.08937962353229523, "p95": 0.17139491438865662 }, { "label": "current_group_partial", "mean": 0.3706088662147522, "median": 0.3705657124519348, "p05": 0.282781720161438, "p95": 0.4561466872692108 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.594892978668213, "group": 6, "latest_source_mass": 0.4636557698249817, "layer": 23, "max_source_mass": 0.4636557698249817, "mixer_index": 45, "normalized_entropy": 0.8196128579947488, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10721131414175034, "median": 0.10240983963012695, "p05": 0.05563424155116081, "p95": 0.18401460349559784 }, { "label": "completed_group_1", "mean": 0.053925663232803345, "median": 0.0527028888463974, "p05": 0.04038037359714508, "p95": 0.07054222375154495 }, { "label": "completed_group_2", "mean": 0.07673212885856628, "median": 0.07705660909414291, "p05": 0.05211106687784195, "p95": 0.10031480342149734 }, { "label": "completed_group_3", "mean": 0.08865120261907578, "median": 0.08699032664299011, "p05": 0.06242028623819351, "p95": 0.11899784207344055 }, { "label": "completed_group_4", "mean": 0.06888366490602493, "median": 0.06751501560211182, "p05": 0.04784276708960533, "p95": 0.09609999507665634 }, { "label": "completed_group_5", "mean": 0.14094024896621704, "median": 0.13063763082027435, "p05": 0.07297889143228531, "p95": 0.24012833833694458 }, { "label": "current_group_partial", "mean": 0.4636557698249817, "median": 0.46312135457992554, "p05": 0.31259939074516296, "p95": 0.6109440326690674 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7775511741638184, "group": 6, "latest_source_mass": 0.3569435477256775, "layer": 24, "max_source_mass": 0.3569435477256775, "mixer_index": 46, "normalized_entropy": 0.9134806018801903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13637229800224304, "median": 0.13456633687019348, "p05": 0.09586701542139053, "p95": 0.18876218795776367 }, { "label": "completed_group_1", "mean": 0.10478455573320389, "median": 0.10326151549816132, "p05": 0.0880400538444519, "p95": 0.12566551566123962 }, { "label": "completed_group_2", "mean": 0.07710614800453186, "median": 0.07678486406803131, "p05": 0.06292064487934113, "p95": 0.09218747913837433 }, { "label": "completed_group_3", "mean": 0.08548048138618469, "median": 0.08430111408233643, "p05": 0.0665682703256607, "p95": 0.10857507586479187 }, { "label": "completed_group_4", "mean": 0.11000140011310577, "median": 0.10904969274997711, "p05": 0.0881001353263855, "p95": 0.13561299443244934 }, { "label": "completed_group_5", "mean": 0.12931162118911743, "median": 0.12692990899085999, "p05": 0.08648453652858734, "p95": 0.1814313679933548 }, { "label": "current_group_partial", "mean": 0.3569435477256775, "median": 0.3568228483200073, "p05": 0.26841166615486145, "p95": 0.4493965804576874 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.654036045074463, "group": 6, "latest_source_mass": 0.4265251159667969, "layer": 24, "max_source_mass": 0.4265251159667969, "mixer_index": 47, "normalized_entropy": 0.8500063817835848, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10232305526733398, "median": 0.08623291552066803, "p05": 0.04679499566555023, "p95": 0.23077142238616943 }, { "label": "completed_group_1", "mean": 0.061968062072992325, "median": 0.060845747590065, "p05": 0.0479469932615757, "p95": 0.07941868156194687 }, { "label": "completed_group_2", "mean": 0.08900615572929382, "median": 0.08864577114582062, "p05": 0.0641680359840393, "p95": 0.11413820832967758 }, { "label": "completed_group_3", "mean": 0.1005433052778244, "median": 0.09813467413187027, "p05": 0.0731433779001236, "p95": 0.1343638002872467 }, { "label": "completed_group_4", "mean": 0.07864196598529816, "median": 0.07639941573143005, "p05": 0.05487418174743652, "p95": 0.10941869765520096 }, { "label": "completed_group_5", "mean": 0.14099235832691193, "median": 0.12885531783103943, "p05": 0.07315157353878021, "p95": 0.24481485784053802 }, { "label": "current_group_partial", "mean": 0.4265251159667969, "median": 0.42903727293014526, "p05": 0.2803359627723694, "p95": 0.5635722875595093 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.84956693649292, "group": 7, "latest_source_mass": 0.28775152564048767, "layer": 25, "max_source_mass": 0.28775152564048767, "mixer_index": 48, "normalized_entropy": 0.9504893827656096, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14851760864257812, "median": 0.14813844859600067, "p05": 0.10312555730342865, "p95": 0.20065288245677948 }, { "label": "completed_group_1", "mean": 0.11630287766456604, "median": 0.11500488221645355, "p05": 0.09825178980827332, "p95": 0.13750022649765015 }, { "label": "completed_group_2", "mean": 0.08512166142463684, "median": 0.08458003401756287, "p05": 0.0710662305355072, "p95": 0.10092862695455551 }, { "label": "completed_group_3", "mean": 0.09215595573186874, "median": 0.09079311043024063, "p05": 0.07304700464010239, "p95": 0.11515989899635315 }, { "label": "completed_group_4", "mean": 0.1263950914144516, "median": 0.12563186883926392, "p05": 0.10441911220550537, "p95": 0.1519687920808792 }, { "label": "completed_group_5", "mean": 0.14375528693199158, "median": 0.1422564834356308, "p05": 0.10313796997070312, "p95": 0.1929876059293747 }, { "label": "completed_group_6", "mean": 0.28775152564048767, "median": 0.28676605224609375, "p05": 0.21690022945404053, "p95": 0.3624127507209778 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7161309719085693, "group": 7, "latest_source_mass": 0.32536065578460693, "layer": 25, "max_source_mass": 0.32536065578460693, "mixer_index": 49, "normalized_entropy": 0.8252845475628168, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0837959572672844, "median": 0.0849243700504303, "p05": 0.0271806251257658, "p95": 0.15180703997612 }, { "label": "completed_group_1", "mean": 0.04434927552938461, "median": 0.04302988201379776, "p05": 0.03217232972383499, "p95": 0.06002792343497276 }, { "label": "completed_group_2", "mean": 0.056708935648202896, "median": 0.05563671886920929, "p05": 0.03992946073412895, "p95": 0.07731831818819046 }, { "label": "completed_group_3", "mean": 0.06495873630046844, "median": 0.06223570555448532, "p05": 0.04220382124185562, "p95": 0.0949486792087555 }, { "label": "completed_group_4", "mean": 0.04885463789105415, "median": 0.046898528933525085, "p05": 0.030923232436180115, "p95": 0.0743112787604332 }, { "label": "completed_group_5", "mean": 0.09997115284204483, "median": 0.08650931715965271, "p05": 0.04111029952764511, "p95": 0.20292627811431885 }, { "label": "completed_group_6", "mean": 0.2760006785392761, "median": 0.27142390608787537, "p05": 0.13761089742183685, "p95": 0.42605364322662354 }, { "label": "current_group_partial", "mean": 0.32536065578460693, "median": 0.3348004221916199, "p05": 0.10654139518737793, "p95": 0.5236068964004517 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9285151958465576, "group": 7, "latest_source_mass": 0.2804902195930481, "layer": 26, "max_source_mass": 0.2804902195930481, "mixer_index": 50, "normalized_entropy": 0.9274197697756124, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10564712435007095, "median": 0.10497278720140457, "p05": 0.04608207941055298, "p95": 0.16350959241390228 }, { "label": "completed_group_1", "mean": 0.09173665940761566, "median": 0.09041133522987366, "p05": 0.07509034872055054, "p95": 0.11238400638103485 }, { "label": "completed_group_2", "mean": 0.07135222852230072, "median": 0.06981296837329865, "p05": 0.0522611029446125, "p95": 0.09555281698703766 }, { "label": "completed_group_3", "mean": 0.07045470923185349, "median": 0.06927657127380371, "p05": 0.055566973984241486, "p95": 0.08908769488334656 }, { "label": "completed_group_4", "mean": 0.09785081446170807, "median": 0.0971219539642334, "p05": 0.07590530812740326, "p95": 0.12285276502370834 }, { "label": "completed_group_5", "mean": 0.10425601899623871, "median": 0.10151226818561554, "p05": 0.06786561012268066, "p95": 0.15156632661819458 }, { "label": "completed_group_6", "mean": 0.1782122254371643, "median": 0.1762663722038269, "p05": 0.11771149188280106, "p95": 0.24932436645030975 }, { "label": "current_group_partial", "mean": 0.2804902195930481, "median": 0.2783043086528778, "p05": 0.14811429381370544, "p95": 0.4168073534965515 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7219383716583252, "group": 7, "latest_source_mass": 0.3661728501319885, "layer": 26, "max_source_mass": 0.3661728501319885, "mixer_index": 51, "normalized_entropy": 0.8280773165026276, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0923302173614502, "median": 0.08522969484329224, "p05": 0.056062981486320496, "p95": 0.1509525179862976 }, { "label": "completed_group_1", "mean": 0.04646994173526764, "median": 0.04543843865394592, "p05": 0.032575178891420364, "p95": 0.06306323409080505 }, { "label": "completed_group_2", "mean": 0.05750556290149689, "median": 0.05649096518754959, "p05": 0.03895717114210129, "p95": 0.07940060645341873 }, { "label": "completed_group_3", "mean": 0.06549686938524246, "median": 0.062135741114616394, "p05": 0.04169235751032829, "p95": 0.09948549419641495 }, { "label": "completed_group_4", "mean": 0.054404959082603455, "median": 0.05241036415100098, "p05": 0.03378590941429138, "p95": 0.08157843351364136 }, { "label": "completed_group_5", "mean": 0.08427833020687103, "median": 0.07576382160186768, "p05": 0.041103728115558624, "p95": 0.15204203128814697 }, { "label": "completed_group_6", "mean": 0.2333412617444992, "median": 0.22215056419372559, "p05": 0.10060560703277588, "p95": 0.3983197808265686 }, { "label": "current_group_partial", "mean": 0.3661728501319885, "median": 0.3673480749130249, "p05": 0.16410952806472778, "p95": 0.5635213255882263 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9470824003219604, "group": 7, "latest_source_mass": 0.26050227880477905, "layer": 27, "max_source_mass": 0.26050227880477905, "mixer_index": 52, "normalized_entropy": 0.9363487077155573, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10804469138383865, "median": 0.10379163920879364, "p05": 0.05421917140483856, "p95": 0.17028658092021942 }, { "label": "completed_group_1", "mean": 0.07850409299135208, "median": 0.0776081532239914, "p05": 0.063654825091362, "p95": 0.09574449062347412 }, { "label": "completed_group_2", "mean": 0.07298290729522705, "median": 0.07200999557971954, "p05": 0.05533483251929283, "p95": 0.09299935400485992 }, { "label": "completed_group_3", "mean": 0.08511961251497269, "median": 0.08372659981250763, "p05": 0.06449062377214432, "p95": 0.11086944490671158 }, { "label": "completed_group_4", "mean": 0.09968686103820801, "median": 0.09877584874629974, "p05": 0.07606595754623413, "p95": 0.1266063004732132 }, { "label": "completed_group_5", "mean": 0.11502514779567719, "median": 0.11171717941761017, "p05": 0.07302433997392654, "p95": 0.1681303083896637 }, { "label": "completed_group_6", "mean": 0.18013443052768707, "median": 0.1765173077583313, "p05": 0.11514104157686234, "p95": 0.2599799633026123 }, { "label": "current_group_partial", "mean": 0.26050227880477905, "median": 0.2575443983078003, "p05": 0.13687226176261902, "p95": 0.39756521582603455 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7855088710784912, "group": 7, "latest_source_mass": 0.28804832696914673, "layer": 27, "max_source_mass": 0.28804832696914673, "mixer_index": 53, "normalized_entropy": 0.858648264589397, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08493785560131073, "median": 0.05951457470655441, "p05": 0.02441210113465786, "p95": 0.22703105211257935 }, { "label": "completed_group_1", "mean": 0.05972691997885704, "median": 0.058127306401729584, "p05": 0.03591413050889969, "p95": 0.08903861790895462 }, { "label": "completed_group_2", "mean": 0.07017223536968231, "median": 0.06906847655773163, "p05": 0.04151676222681999, "p95": 0.10273467004299164 }, { "label": "completed_group_3", "mean": 0.07963316142559052, "median": 0.07547932863235474, "p05": 0.042773112654685974, "p95": 0.12874338030815125 }, { "label": "completed_group_4", "mean": 0.0645163506269455, "median": 0.061876002699136734, "p05": 0.0366181954741478, "p95": 0.10098200291395187 }, { "label": "completed_group_5", "mean": 0.11586236953735352, "median": 0.10182450711727142, "p05": 0.048887670040130615, "p95": 0.2313569337129593 }, { "label": "completed_group_6", "mean": 0.23710274696350098, "median": 0.22610372304916382, "p05": 0.10011275857686996, "p95": 0.4084784686565399 }, { "label": "current_group_partial", "mean": 0.28804832696914673, "median": 0.27438053488731384, "p05": 0.07688213139772415, "p95": 0.5529112815856934 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.973182201385498, "group": 7, "latest_source_mass": 0.23531918227672577, "layer": 28, "max_source_mass": 0.23531918227672577, "mixer_index": 54, "normalized_entropy": 0.9489000589030754, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.104981929063797, "median": 0.09386023879051208, "p05": 0.04212673753499985, "p95": 0.18614700436592102 }, { "label": "completed_group_1", "mean": 0.0840599462389946, "median": 0.08250434696674347, "p05": 0.05170809105038643, "p95": 0.12236068397760391 }, { "label": "completed_group_2", "mean": 0.09310521930456161, "median": 0.0931512713432312, "p05": 0.06374019384384155, "p95": 0.12329407781362534 }, { "label": "completed_group_3", "mean": 0.11566174775362015, "median": 0.11439734697341919, "p05": 0.07886623591184616, "p95": 0.1571493148803711 }, { "label": "completed_group_4", "mean": 0.0963074117898941, "median": 0.09477876126766205, "p05": 0.06686149537563324, "p95": 0.13022682070732117 }, { "label": "completed_group_5", "mean": 0.1096896305680275, "median": 0.10702179372310638, "p05": 0.06579374521970749, "p95": 0.1643676459789276 }, { "label": "completed_group_6", "mean": 0.16087493300437927, "median": 0.15455034375190735, "p05": 0.09876372665166855, "p95": 0.24324838817119598 }, { "label": "current_group_partial", "mean": 0.23531918227672577, "median": 0.22382870316505432, "p05": 0.11656329780817032, "p95": 0.38286590576171875 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.802741289138794, "group": 7, "latest_source_mass": 0.2497689425945282, "layer": 28, "max_source_mass": 0.2913346588611603, "mixer_index": 55, "normalized_entropy": 0.8669353059487718, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08190082013607025, "median": 0.06415966153144836, "p05": 0.039168842136859894, "p95": 0.23471535742282867 }, { "label": "completed_group_1", "mean": 0.05632811039686203, "median": 0.055233441293239594, "p05": 0.03936203569173813, "p95": 0.076178178191185 }, { "label": "completed_group_2", "mean": 0.07102417945861816, "median": 0.06932097673416138, "p05": 0.04159427434206009, "p95": 0.1046731173992157 }, { "label": "completed_group_3", "mean": 0.08421063423156738, "median": 0.0812690332531929, "p05": 0.0486646331846714, "p95": 0.12889066338539124 }, { "label": "completed_group_4", "mean": 0.06341387331485748, "median": 0.061601825058460236, "p05": 0.03807957097887993, "p95": 0.09577237069606781 }, { "label": "completed_group_5", "mean": 0.10201875865459442, "median": 0.09330011904239655, "p05": 0.05116701498627663, "p95": 0.18108239769935608 }, { "label": "completed_group_6", "mean": 0.2913346588611603, "median": 0.28823426365852356, "p05": 0.13295966386795044, "p95": 0.4586564898490906 }, { "label": "current_group_partial", "mean": 0.2497689425945282, "median": 0.234481081366539, "p05": 0.09318265318870544, "p95": 0.4530518054962158 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.945815086364746, "group": 8, "latest_source_mass": 0.153213769197464, "layer": 29, "max_source_mass": 0.2451319396495819, "mixer_index": 56, "normalized_entropy": 0.9357392585284499, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1064736545085907, "median": 0.0881282389163971, "p05": 0.029344500973820686, "p95": 0.23970629274845123 }, { "label": "completed_group_1", "mean": 0.07925166189670563, "median": 0.07691921293735504, "p05": 0.062442928552627563, "p95": 0.1030145138502121 }, { "label": "completed_group_2", "mean": 0.07823626697063446, "median": 0.07715678215026855, "p05": 0.059018202126026154, "p95": 0.10055352747440338 }, { "label": "completed_group_3", "mean": 0.09447507560253143, "median": 0.09318602085113525, "p05": 0.0684189647436142, "p95": 0.12381350249052048 }, { "label": "completed_group_4", "mean": 0.09724453091621399, "median": 0.09628183394670486, "p05": 0.0699525699019432, "p95": 0.12776269018650055 }, { "label": "completed_group_5", "mean": 0.1459731012582779, "median": 0.14035329222679138, "p05": 0.08667999505996704, "p95": 0.2258891463279724 }, { "label": "completed_group_6", "mean": 0.2451319396495819, "median": 0.24505369365215302, "p05": 0.16207969188690186, "p95": 0.3307104706764221 }, { "label": "completed_group_7", "mean": 0.153213769197464, "median": 0.13417893648147583, "p05": 0.05588865652680397, "p95": 0.30438560247421265 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8717057704925537, "group": 8, "latest_source_mass": 0.3048541843891144, "layer": 29, "max_source_mass": 0.3048541843891144, "mixer_index": 57, "normalized_entropy": 0.8518500065030654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07336360216140747, "median": 0.0611552819609642, "p05": 0.024239415302872658, "p95": 0.16916418075561523 }, { "label": "completed_group_1", "mean": 0.05720400810241699, "median": 0.05402737483382225, "p05": 0.0314149409532547, "p95": 0.09488057345151901 }, { "label": "completed_group_2", "mean": 0.057025715708732605, "median": 0.05547948181629181, "p05": 0.03480810672044754, "p95": 0.08415934443473816 }, { "label": "completed_group_3", "mean": 0.06655597686767578, "median": 0.06265220046043396, "p05": 0.035200610756874084, "p95": 0.10949824750423431 }, { "label": "completed_group_4", "mean": 0.05032068490982056, "median": 0.047904569655656815, "p05": 0.02614513412117958, "p95": 0.08244554698467255 }, { "label": "completed_group_5", "mean": 0.08329877257347107, "median": 0.0731179416179657, "p05": 0.02939312346279621, "p95": 0.17248544096946716 }, { "label": "completed_group_6", "mean": 0.1901058554649353, "median": 0.18234091997146606, "p05": 0.08719424903392792, "p95": 0.317789226770401 }, { "label": "completed_group_7", "mean": 0.11727118492126465, "median": 0.09272899478673935, "p05": 0.031939513981342316, "p95": 0.2891285717487335 }, { "label": "current_group_partial", "mean": 0.3048541843891144, "median": 0.29432767629623413, "p05": 0.07913826406002045, "p95": 0.5632759928703308 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.09096622467041, "group": 8, "latest_source_mass": 0.13795366883277893, "layer": 30, "max_source_mass": 0.17320571839809418, "mixer_index": 58, "normalized_entropy": 0.951639739623416, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10326676070690155, "median": 0.09009090811014175, "p05": 0.01757553033530712, "p95": 0.2674753665924072 }, { "label": "completed_group_1", "mean": 0.08232800662517548, "median": 0.08093428611755371, "p05": 0.06430500745773315, "p95": 0.10457758605480194 }, { "label": "completed_group_2", "mean": 0.07164021581411362, "median": 0.0711187869310379, "p05": 0.05190654471516609, "p95": 0.09468771517276764 }, { "label": "completed_group_3", "mean": 0.08822821825742722, "median": 0.08667590469121933, "p05": 0.06148773431777954, "p95": 0.12025904655456543 }, { "label": "completed_group_4", "mean": 0.10803373157978058, "median": 0.10627314448356628, "p05": 0.07042569667100906, "p95": 0.1514367014169693 }, { "label": "completed_group_5", "mean": 0.13578416407108307, "median": 0.12929600477218628, "p05": 0.08227657526731491, "p95": 0.21117588877677917 }, { "label": "completed_group_6", "mean": 0.17320571839809418, "median": 0.1694163680076599, "p05": 0.11322733014822006, "p95": 0.24482715129852295 }, { "label": "completed_group_7", "mean": 0.09955951571464539, "median": 0.08973086625337601, "p05": 0.037010837346315384, "p95": 0.1912725567817688 }, { "label": "current_group_partial", "mean": 0.13795366883277893, "median": 0.1256815791130066, "p05": 0.053015124052762985, "p95": 0.2601332366466522 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.96881103515625, "group": 8, "latest_source_mass": 0.21570813655853271, "layer": 30, "max_source_mass": 0.21570813655853271, "mixer_index": 59, "normalized_entropy": 0.896044517007504, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08843819797039032, "median": 0.08496083319187164, "p05": 0.03690534085035324, "p95": 0.1522325873374939 }, { "label": "completed_group_1", "mean": 0.06052382290363312, "median": 0.058369796723127365, "p05": 0.03549504280090332, "p95": 0.09154202789068222 }, { "label": "completed_group_2", "mean": 0.0642135739326477, "median": 0.06355445086956024, "p05": 0.03948400169610977, "p95": 0.09175601601600647 }, { "label": "completed_group_3", "mean": 0.07744920998811722, "median": 0.07566337287425995, "p05": 0.04564609378576279, "p95": 0.11733324825763702 }, { "label": "completed_group_4", "mean": 0.051616813987493515, "median": 0.049985647201538086, "p05": 0.03238597884774208, "p95": 0.07699409872293472 }, { "label": "completed_group_5", "mean": 0.06631231307983398, "median": 0.061356961727142334, "p05": 0.030705859884619713, "p95": 0.1188126802444458 }, { "label": "completed_group_6", "mean": 0.16417551040649414, "median": 0.15454982221126556, "p05": 0.07227650284767151, "p95": 0.28998225927352905 }, { "label": "completed_group_7", "mean": 0.2115624099969864, "median": 0.19997704029083252, "p05": 0.09131578356027603, "p95": 0.3709705173969269 }, { "label": "current_group_partial", "mean": 0.21570813655853271, "median": 0.2005728930234909, "p05": 0.06839637458324432, "p95": 0.4193759858608246 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.124063014984131, "group": 8, "latest_source_mass": 0.1258423924446106, "layer": 31, "max_source_mass": 0.17017140984535217, "mixer_index": 60, "normalized_entropy": 0.9667027380329118, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09220977127552032, "median": 0.09152225404977798, "p05": 0.03251190483570099, "p95": 0.17071731388568878 }, { "label": "completed_group_1", "mean": 0.06839152425527573, "median": 0.06702418625354767, "p05": 0.04446800425648689, "p95": 0.0977444052696228 }, { "label": "completed_group_2", "mean": 0.10039852559566498, "median": 0.09736946225166321, "p05": 0.06476961821317673, "p95": 0.14545989036560059 }, { "label": "completed_group_3", "mean": 0.10902463644742966, "median": 0.10749617218971252, "p05": 0.07769118249416351, "p95": 0.14509552717208862 }, { "label": "completed_group_4", "mean": 0.0849665105342865, "median": 0.08355648815631866, "p05": 0.059977494180202484, "p95": 0.11480176448822021 }, { "label": "completed_group_5", "mean": 0.12137698382139206, "median": 0.11852307617664337, "p05": 0.07902064919471741, "p95": 0.17207489907741547 }, { "label": "completed_group_6", "mean": 0.17017140984535217, "median": 0.16817378997802734, "p05": 0.11521546542644501, "p95": 0.23236754536628723 }, { "label": "completed_group_7", "mean": 0.12761825323104858, "median": 0.1237221211194992, "p05": 0.062459878623485565, "p95": 0.2092883139848709 }, { "label": "current_group_partial", "mean": 0.1258423924446106, "median": 0.11723431199789047, "p05": 0.06504593789577484, "p95": 0.21421310305595398 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9633058309555054, "group": 8, "latest_source_mass": 0.15763342380523682, "layer": 31, "max_source_mass": 0.23511825501918793, "mixer_index": 61, "normalized_entropy": 0.8935389906004497, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.0846441388130188, "median": 0.06999644637107849, "p05": 0.04030589759349823, "p95": 0.20743247866630554 }, { "label": "completed_group_1", "mean": 0.05872901901602745, "median": 0.05653517693281174, "p05": 0.03710630163550377, "p95": 0.08772030472755432 }, { "label": "completed_group_2", "mean": 0.07210008800029755, "median": 0.07183568179607391, "p05": 0.03840115666389465, "p95": 0.10702595114707947 }, { "label": "completed_group_3", "mean": 0.07792744785547256, "median": 0.0762966051697731, "p05": 0.04050692170858383, "p95": 0.122304767370224 }, { "label": "completed_group_4", "mean": 0.059186771512031555, "median": 0.057271093130111694, "p05": 0.03537197411060333, "p95": 0.08958204090595245 }, { "label": "completed_group_5", "mean": 0.07445863634347916, "median": 0.06928195059299469, "p05": 0.03154807910323143, "p95": 0.1351260244846344 }, { "label": "completed_group_6", "mean": 0.23511825501918793, "median": 0.22703276574611664, "p05": 0.0953572615981102, "p95": 0.3982555568218231 }, { "label": "completed_group_7", "mean": 0.18020221590995789, "median": 0.16846294701099396, "p05": 0.07464539259672165, "p95": 0.324230432510376 }, { "label": "current_group_partial", "mean": 0.15763342380523682, "median": 0.12027859687805176, "p05": 0.036883872002363205, "p95": 0.4093724191188812 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1238560676574707, "group": 8, "latest_source_mass": 0.11516246944665909, "layer": 32, "max_source_mass": 0.1766587197780609, "mixer_index": 62, "normalized_entropy": 0.966608552245626, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10231245309114456, "median": 0.10458895564079285, "p05": 0.025718798860907555, "p95": 0.1875312179327011 }, { "label": "completed_group_1", "mean": 0.07575537264347076, "median": 0.07447351515293121, "p05": 0.05878028646111488, "p95": 0.09651422500610352 }, { "label": "completed_group_2", "mean": 0.08897508680820465, "median": 0.08751188218593597, "p05": 0.06392349302768707, "p95": 0.11800213158130646 }, { "label": "completed_group_3", "mean": 0.0967075377702713, "median": 0.09579810500144958, "p05": 0.06915527582168579, "p95": 0.12801720201969147 }, { "label": "completed_group_4", "mean": 0.09801244735717773, "median": 0.09654392302036285, "p05": 0.07431379705667496, "p95": 0.12754815816879272 }, { "label": "completed_group_5", "mean": 0.12438583374023438, "median": 0.12010829150676727, "p05": 0.08193086087703705, "p95": 0.1824759691953659 }, { "label": "completed_group_6", "mean": 0.1766587197780609, "median": 0.17520052194595337, "p05": 0.12216726690530777, "p95": 0.2341049611568451 }, { "label": "completed_group_7", "mean": 0.12203006446361542, "median": 0.1200856938958168, "p05": 0.05817953124642372, "p95": 0.19647708535194397 }, { "label": "current_group_partial", "mean": 0.11516246944665909, "median": 0.10430245846509933, "p05": 0.05224763602018356, "p95": 0.2294922173023224 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0835556983947754, "group": 8, "latest_source_mass": 0.09660109132528305, "layer": 32, "max_source_mass": 0.22511571645736694, "mixer_index": 63, "normalized_entropy": 0.9482670637704002, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09133295714855194, "median": 0.08019149303436279, "p05": 0.04441600665450096, "p95": 0.19931957125663757 }, { "label": "completed_group_1", "mean": 0.07054752856492996, "median": 0.06906121969223022, "p05": 0.05099350959062576, "p95": 0.09504306316375732 }, { "label": "completed_group_2", "mean": 0.09000501036643982, "median": 0.0901019275188446, "p05": 0.05998067185282707, "p95": 0.11849517375230789 }, { "label": "completed_group_3", "mean": 0.10956105589866638, "median": 0.1064089685678482, "p05": 0.0759081020951271, "p95": 0.1509535312652588 }, { "label": "completed_group_4", "mean": 0.07114207744598389, "median": 0.06995616853237152, "p05": 0.04991999268531799, "p95": 0.09545011073350906 }, { "label": "completed_group_5", "mean": 0.11565887182950974, "median": 0.11069735884666443, "p05": 0.06978912651538849, "p95": 0.17647549510002136 }, { "label": "completed_group_6", "mean": 0.22511571645736694, "median": 0.22372552752494812, "p05": 0.11895382404327393, "p95": 0.33412501215934753 }, { "label": "completed_group_7", "mean": 0.13003569841384888, "median": 0.12226496636867523, "p05": 0.06884703785181046, "p95": 0.21555939316749573 }, { "label": "current_group_partial", "mean": 0.09660109132528305, "median": 0.09000182151794434, "p05": 0.051288168877363205, "p95": 0.16580010950565338 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.679484486579895, "group": null, "latest_source_mass": 0.2827349305152893, "layer": null, "max_source_mass": 0.32717907428741455, "mixer_index": null, "normalized_entropy": 0.7643663300981273, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.08993907272815704, "median": 0.09290459752082825, "p05": 0.022419869899749756, "p95": 0.15114322304725647 }, { "label": "completed_group_1", "mean": 0.043493133038282394, "median": 0.042318835854530334, "p05": 0.026918282732367516, "p95": 0.06356139481067657 }, { "label": "completed_group_2", "mean": 0.04922422394156456, "median": 0.047891765832901, "p05": 0.03095712885260582, "p95": 0.07266643643379211 }, { "label": "completed_group_3", "mean": 0.037924449890851974, "median": 0.03562554344534874, "p05": 0.020909149199724197, "p95": 0.06224166974425316 }, { "label": "completed_group_4", "mean": 0.03939433768391609, "median": 0.036868683993816376, "p05": 0.020651251077651978, "p95": 0.06597590446472168 }, { "label": "completed_group_5", "mean": 0.03898489475250244, "median": 0.03362073004245758, "p05": 0.014276490546762943, "p95": 0.08282073587179184 }, { "label": "completed_group_6", "mean": 0.09112587571144104, "median": 0.07136045396327972, "p05": 0.02230823040008545, "p95": 0.22286513447761536 }, { "label": "completed_group_7", "mean": 0.32717907428741455, "median": 0.3213679790496826, "p05": 0.08271751552820206, "p95": 0.6128557920455933 }, { "label": "completed_group_8", "mean": 0.2827349305152893, "median": 0.2627606987953186, "p05": 0.05262754485011101, "p95": 0.571351170539856 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "optimizer_state_before": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08626964688301086, 0.16337208449840546, 0.15584443509578705, 0.04305505007505417, 0.1360139101743698, 0.03393339365720749, 0.03730965405702591, 0.037494905292987823, 0.05234000086784363, 0.04223497211933136, 0.041619230061769485, 0.05722156912088394, 0.062187470495700836, 0.0369485504925251, 0.04293901473283768, 0.045369699597358704, 0.04149825870990753, 0.02440357394516468, 0.026009807363152504, 0.03634416684508324, 0.038355328142642975, 0.02631118707358837, 0.033921729773283005, 0.02731483057141304, 0.027557097375392914, 0.024315863847732544, 0.02051021158695221, 0.018631720915436745, 0.019497457891702652, 0.021715763956308365, 0.017114246264100075, 0.020576251670718193 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.0936665578119573e-05, "first_to_last_ratio": 1.168787114339494, "imbalance_abs_log_ratio": 0.1559665567011555, "last_quartile_mean": 1.7913155716087203e-05, "mean": 2.42622821247096e-05, "normalized": [ 0.7126129382274812, 0.7437894782846245, 0.7319401693963385, 0.6938624658898646, 1.0850314310673168, 1.0981344950491732, 0.9606310812212513, 0.877442710157735, 1.0089375010128527, 0.8940805493086196, 0.8303081011723588, 0.7704924450335486, 0.7457227037132441, 0.6956545192296728, 0.6629687198417477, 0.6359294291467756, 1.5989410181555679, 1.4898579524033908, 1.348196746912536, 1.1824041858349024, 2.2648751765532618, 2.0078719511613548, 1.669089976380829, 1.384721107610681, 1.3105185686643073, 1.0955274221780484, 0.8401582843314063, 0.6990787108453305, 0.7286997339090275, 0.5032506445515866, 0.392614310962981, 0.33665547179218436 ], "peak_layer": 21, "peak_normalized": 2.2648751765532618, "population_cv": 0.43871254744666655, "rest_mean": 2.099399386713264e-05, "spike_contrast": 1.9963347127217863, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.191103871562518e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.7289616152993403e-05, 1.8046030163532123e-05, 1.77585388883017e-05, 1.6834686903166585e-05, 2.6325338694732636e-05, 2.6643248929758556e-05, 2.330710231035482e-05, 2.128876258211676e-05, 2.4479126295773312e-05, 2.169243452954106e-05, 2.014516940107569e-05, 1.869390507636126e-05, 1.8092934624291956e-05, 1.687816620687954e-05, 1.608513412065804e-05, 1.5429099221364595e-05, 3.8793958083260804e-05, 3.614735396695323e-05, 3.271032983320765e-05, 2.868782394216396e-05, 5.49510405107867e-05, 4.871555574936792e-05, 4.0495931898476556e-05, 3.35964941768907e-05, 3.179617124260403e-05, 2.6579995392239653e-05, 2.0384157323860563e-05, 1.696124490990769e-05, 1.767991852830164e-05, 1.2210009117552545e-05, 9.52571917878231e-06, 8.168030035449192e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 8.209008865378564e-05, "first_to_last_ratio": 1.1473326374136463, "imbalance_abs_log_ratio": 0.13743980256699795, "last_quartile_mean": 7.154863897085306e-05, "mean": 9.638685185109352e-05, "normalized": [ 0.7014672692081079, 0.730745142012726, 0.7197152784749434, 0.6836365522681692, 1.073826467208612, 1.0803379033203697, 0.9514146538604249, 0.8722415119444402, 1.0096074578076277, 0.8908657485913869, 0.8272757726491637, 0.7688961382914924, 0.7419179025068791, 0.6953651992846211, 0.6657440869339828, 0.6349379036282153, 1.6060213837194506, 1.5018800747743897, 1.362846846779478, 1.1942969804897212, 2.2656295070514414, 2.0104706485090547, 1.6756941049043201, 1.3967091226783026, 1.3149112210845808, 1.1000992011684827, 0.8430071951587476, 0.7025118587919716, 0.7359466516142374, 0.5075845580254474, 0.3944123577040649, 0.33998329955514606 ], "peak_layer": 21, "peak_normalized": 2.2656295070514414, "population_cv": 0.4413317448728873, "rest_mean": 8.330888886121102e-05, "spike_contrast": 2.004682264754327, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016700785199645906, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.761222175555304e-05, 7.043422374408692e-05, 6.93710899213329e-05, 6.589357508346438e-05, 0.00010350275260861963, 0.00010413036943646148, 9.17038632906042e-05, 8.407261339016259e-05, 9.731288446346298e-05, 8.586774492869154e-05, 7.973850733833387e-05, 7.411147817038e-05, 7.15111309546046e-05, 6.70240624458529e-05, 6.416897667804733e-05, 6.119966565165669e-05, 0.0001547993451822549, 0.00014476149226538837, 0.0001313605171162635, 0.0001151145261246711, 0.00021837689564563334, 0.00019378293654881418, 0.00016151487943716347, 0.00013462439528666437, 0.00012674015306401998, 0.00010603509872453287, 8.125480962917209e-05, 6.771290645701811e-05, 7.093558087944984e-05, 4.8924477596301585e-05, 3.801616549026221e-05, 3.2769919926067814e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.07279830891638994, "first_to_last_ratio": 1.1473325978581586, "imbalance_abs_log_ratio": 0.13743976809095573, "last_quartile_mean": 0.06345004844479263, "mean": 0.08547682355856523, "normalized": [ 0.701467243783159, 0.7307451127750545, 0.7197152636360212, 0.6836365259909346, 1.0738264172711294, 1.080337898385664, 0.951414616770227, 0.8722414984608238, 1.0096074000217996, 0.8908656785314788, 0.8272757305190417, 0.7688961528411616, 0.7419179123849904, 0.6953651763038863, 0.6657440954344426, 0.6349378767127872, 1.6060213451352348, 1.5018801837842675, 1.3628468172791657, 1.19429705497508, 2.2656295095184866, 2.010470763800267, 1.6756941843442625, 1.3967091688883524, 1.314911253304382, 1.1000991448544435, 0.8430072555120299, 0.7025118281998033, 0.7359466794525621, 0.5075845660770223, 0.3944123343969408, 0.3399833106550978 ], "peak_layer": 21, "peak_normalized": 2.2656295095184866, "population_cv": 0.4413317644266601, "rest_mean": 0.07387915439903736, "spike_contrast": 2.0046823522109167, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1481042370200157, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.05995919182896614, 0.062461771070957184, 0.06151897460222244, 0.05843507871031761, 0.09178727120161057, 0.09234385192394257, 0.08132389932870865, 0.07455643266439438, 0.08629803359508514, 0.0761483684182167, 0.07071290165185928, 0.06572280079126358, 0.06341678649187088, 0.05943760648369789, 0.05690569058060646, 0.054272472858428955, 0.13727760314941406, 0.12837594747543335, 0.11649181693792343, 0.10208471864461899, 0.19365881383419037, 0.17184865474700928, 0.1432330161333084, 0.11938626319169998, 0.11239443719387054, 0.09403298050165176, 0.0720575824379921, 0.060048479586839676, 0.06290638446807861, 0.04338671639561653, 0.03371311351656914, 0.029060693457722664 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.120508836218505e-06, "first_to_last_ratio": 0.6479745229308365, "imbalance_abs_log_ratio": 0.43390389986753336, "last_quartile_mean": 4.815789395706815e-06, "mean": 5.18821034489747e-06, "normalized": [ 0.6392329797809361, 0.5994300881226403, 0.5850824179653978, 0.6009191324861043, 0.6062512409598932, 0.6024392926484339, 0.5897072336534216, 0.5886297066862235, 0.7270901470558813, 0.6920316696984324, 0.6762839200674405, 0.6775578709931118, 0.6128406906582687, 0.6178016007991107, 0.624771802695437, 0.6621615984140956, 1.5142793639941121, 1.477704889855872, 1.4442885378272925, 1.3463135645628785, 2.4904030104436794, 2.3638034305258637, 2.088000001391446, 1.747233095035389, 1.6663782972572876, 1.4047981189097183, 1.0886105900542489, 0.8884508074141, 0.8978880977348557, 0.6001555244868991, 0.4703385709663715, 0.40912270685515684 ], "peak_layer": 21, "peak_normalized": 2.4904030104436794, "population_cv": 0.5663270603947061, "rest_mean": 4.159058141373896e-06, "spike_contrast": 2.5836696383318847, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0745632243924774e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.316475158499088e-06, 3.109969384240685e-06, 3.0355306535057025e-06, 3.1176948596112197e-06, 3.145358959955047e-06, 3.125581770291319e-06, 3.0595251701015513e-06, 3.0539347335434286e-06, 3.772296622628346e-06, 3.5904058677260764e-06, 3.508703230181709e-06, 3.5153127555531682e-06, 3.1795464110473404e-06, 3.2052846563601634e-06, 3.2414475299447076e-06, 3.4354336548858555e-06, 7.856399861339014e-06, 7.666643796255812e-06, 7.4932727329724e-06, 6.984957963140914e-06, 1.2920734661747701e-05, 1.2263909411558416e-05, 1.0832983207365032e-05, 9.06501281860983e-06, 8.645521120342892e-06, 7.288388133019907e-06, 5.647940724884393e-06, 4.609469669958344e-06, 4.658432317228289e-06, 3.113733100690297e-06, 2.4402154394920217e-06, 2.12261466003838e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.9889651311095804e-05, "first_to_last_ratio": 1.3326615656169287, "imbalance_abs_log_ratio": 0.28717811970161894, "last_quartile_mean": 4.493988035392249e-05, "mean": 6.640704737037595e-05, "normalized": [ 0.7737542671823434, 0.7960594489342157, 0.7852142182091284, 0.752976874377423, 1.1002658507118863, 1.0894970423039605, 0.9958150050309817, 0.921272183824516, 1.0695745385738102, 0.9470829860046872, 0.8771170672423568, 0.8087454440169668, 0.8138319929245809, 0.7582700648479997, 0.7300932386851152, 0.6932937290424763, 1.5635426486844544, 1.4635271461161115, 1.3107699454980482, 1.1481322979739153, 2.32725688601457, 1.991760393656834, 1.584233429529925, 1.2840444859649542, 1.264803263909513, 1.026616211913306, 0.7846647444878888, 0.6569396053772728, 0.6050973350000657, 0.42438660382108917, 0.34340864002846966, 0.3079524101111341 ], "peak_layer": 21, "peak_normalized": 2.32725688601457, "population_cv": 0.43279320435052254, "rest_mean": 5.791654122596675e-05, "spike_contrast": 1.9382335024498314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011225578055018559, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 5.1382736273808405e-05, 5.286395753500983e-05, 5.2143757784506306e-05, 5.000297096557915e-05, 7.306540646823123e-05, 7.23502816981636e-05, 6.612913421122357e-05, 6.117896555224434e-05, 7.102728704921901e-05, 6.289298471529037e-05, 5.8246754633728415e-05, 5.3706397011410445e-05, 5.4044179705670103e-05, 5.035447611589916e-05, 4.8483336286153644e-05, 4.6039589506108314e-05, 0.00010383025073679164, 9.718851651996374e-05, 8.704436186235398e-05, 7.624407589901239e-05, 0.00015454625827260315, 0.00013226692681200802, 0.00010520426440052688, 8.526960300514475e-05, 8.399185026064515e-05, 6.817455141572282e-05, 5.2107268857071176e-05, 4.362541949376464e-05, 4.018272738903761e-05, 2.8182261303300038e-05, 2.2804753825766966e-05, 2.0450210286071524e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.414140539665823e-05, "first_to_last_ratio": 1.6936904784541074, "imbalance_abs_log_ratio": 0.5269098631562702, "last_quartile_mean": 2.606226223633712e-05, "mean": 4.478222555803768e-05, "normalized": [ 0.874026545726401, 0.9009891441910284, 0.8916727093699365, 0.8527000158254343, 1.172545810112114, 1.156435027941858, 1.0621486326987206, 0.9750045233682423, 1.1460449634090804, 1.011287374798366, 0.9322859052145132, 0.8551582509249812, 0.907118809465358, 0.8439303067882139, 0.8153240377760277, 0.7685666352135386, 1.5553576798550997, 1.437596357968014, 1.2796321689041579, 1.1209246602381584, 2.385838207411615, 1.9252114173862964, 1.4016273255130902, 1.0727508885758477, 1.1785651553842893, 0.9257194279728913, 0.7253410162251672, 0.6191416448731234, 0.43450056827805394, 0.29243769148345633, 0.24680184853049614, 0.23331524857642952 ], "peak_layer": 21, "peak_normalized": 2.385838207411615, "population_cv": 0.43498704004207234, "rest_mean": 3.986614397204162e-05, "spike_contrast": 1.7892140803093455, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.13290661224164e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.914085391443223e-05, 4.034829908050597e-05, 3.9931088394951075e-05, 3.81858044420369e-05, 5.2509210945572704e-05, 5.178773426450789e-05, 4.756537964567542e-05, 4.366287248558365e-05, 5.1322444051038474e-05, 4.528769932221621e-05, 4.1749837691895664e-05, 3.829588968073949e-05, 4.0622799133416265e-05, 3.779307735385373e-05, 3.6512024962576106e-05, 3.441812441451475e-05, 6.965237844269723e-05, 6.437876436393708e-05, 5.7304776419186965e-05, 5.019750096835196e-05, 0.00010684314474929124, 8.621525194030255e-05, 6.276799103943631e-05, 4.804017225978896e-05, 5.277877062326297e-05, 4.1455776226939633e-05, 3.2482384995091707e-05, 2.7726540793082677e-05, 1.9457902453723364e-05, 1.3096010661683977e-05, 1.1052336049033329e-05, 1.0448376087879296e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015771267680975143, "first_to_last_ratio": 1.0243014562120425, "imbalance_abs_log_ratio": 0.024010874124910462, "last_quartile_mean": 0.0001539709583084914, "mean": 0.00019892108525709773, "normalized": [ 0.6590708750981731, 0.6828508221536124, 0.6699110125304764, 0.6473590320765046, 0.9801195462127644, 0.9691430493294225, 0.887896737937672, 0.8463722857737056, 0.988608068348346, 0.8930841762598842, 0.8341812095096794, 0.7796271143503263, 0.7202755659725629, 0.6755421311480478, 0.6527815148816499, 0.632514433083119, 1.5594662296068276, 1.4738278124849744, 1.3649033855466377, 1.1834547507508266, 2.341778688768703, 2.0989567426400018, 1.7777537161382369, 1.4882782565844777, 1.4127403878123095, 1.1574678874298643, 0.8851612824909503, 0.7258447229726291, 0.7423942540292697, 0.5132421599816662, 0.4020767269150468, 0.35331542118163173 ], "peak_layer": 21, "peak_normalized": 2.341778688768703, "population_cv": 0.465450630495743, "rest_mean": 0.00016857082745344896, "spike_contrast": 2.1522850832359604, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00036281247739680114, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.0001311030937358737, 0.000135833426611498, 0.00013325942563824356, 0.00012877336121164262, 0.00019496644381433725, 0.00019278298714198172, 0.0001766213827067986, 0.00016836129361763597, 0.00019665498984977603, 0.00017765327356755733, 0.00016593623149674386, 0.00015508427168242633, 0.0001432779972674325, 0.00013437957386486232, 0.0001298520073760301, 0.00012582045746967196, 0.0003102107148151845, 0.00029317542794160545, 0.00027150806272402406, 0.0002354141033720225, 0.0004658291582018137, 0.00041752675315365195, 0.00035363269853405654, 0.00029604992596432567, 0.0002810238511301577, 0.00023024476831778884, 0.0001760772429406643, 0.00014438582002185285, 0.00014767787070013583, 0.00010209468746325001, 7.998153887456283e-05, 7.028188701951876e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13547764718532562, 0.039667773991823196, 0.026703152805566788, 0.05304057523608208, 0.016798952594399452, 0.00753376679494977, 0.01294450368732214, 0.03526800498366356, 0.030294200405478477, 0.034149833023548126, 0.04367009550333023, 0.05930330231785774, 0.04527241364121437, 0.04281846433877945, 0.04812675341963768, 0.06094835698604584, 0.038031257688999176, 0.040106989443302155, 0.04961257055401802, 0.07085621356964111, 0.03753058984875679, 0.04544757679104805, 0.05368982255458832, 0.06174425780773163, 0.05051252990961075, 0.054937634617090225, 0.05777662619948387, 0.07133159041404724, 0.0636468306183815, 0.09261580556631088, 0.1020733118057251, 0.12878532707691193 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.0824043303946382e-05, "first_to_last_ratio": 1.424267056786447, "imbalance_abs_log_ratio": 0.3536573352845024, "last_quartile_mean": 1.4620883916904859e-05, "mean": 2.2054508917790372e-05, "normalized": [ 0.7842104740080174, 0.8183716989463029, 0.8099804080045728, 0.7638102866038118, 1.1922163664937413, 1.1967612607580402, 1.039219827104321, 0.9490935140140151, 1.019003480899918, 0.9331398258308763, 0.8658119659177055, 0.813018282797004, 0.7774015040255573, 0.7266621576498908, 0.7030285425215664, 0.6829301796443584, 1.632275124747471, 1.4994012377073724, 1.316962971421436, 1.1572238560225663, 2.2864000936278734, 1.8918326711874776, 1.5430845130531994, 1.2946151724909234, 1.217184871626016, 0.9287724220385267, 0.7771760119482759, 0.6578637330922696, 0.5567850124802112, 0.4325353313877837, 0.37087941374039357, 0.3623477882085046 ], "peak_layer": 21, "peak_normalized": 2.2864000936278734, "population_cv": 0.42456281874262314, "rest_mean": 1.9413589850753648e-05, "spike_contrast": 1.870621155539602, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6315471879788676e-05, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.7295376892434433e-05, 1.8048785932478495e-05, 1.7863720131572336e-05, 1.6845460777403787e-05, 2.6293746486771852e-05, 2.6393981897854246e-05, 2.2919482944416814e-05, 2.0931791368639097e-05, 2.247362135676667e-05, 2.0579940610332415e-05, 1.909505772346165e-05, 1.793071896827314e-05, 1.71452084032353e-05, 1.6026177036110312e-05, 1.5504949260503054e-05, 1.5061689737194683e-05, 3.599902629503049e-05, 3.3068557968363166e-05, 2.9044971597613767e-05, 2.552200385252945e-05, 5.042543125455268e-05, 4.1723440517671406e-05, 3.4031971154036e-05, 2.855210186680779e-05, 2.68444146058755e-05, 2.048361966444645e-05, 1.7140235286206007e-05, 1.4508861568174325e-05, 1.2279620023036841e-05, 9.53935432335129e-06, 8.179563337762374e-06, 7.991402526386082e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 8.156287549354602e-05, "first_to_last_ratio": 1.3967979973430424, "imbalance_abs_log_ratio": 0.3341824723557829, "last_quartile_mean": 5.839274945174111e-05, "mean": 8.75740092851629e-05, "normalized": [ 0.7720751985061499, 0.8035956764433113, 0.7960739585260428, 0.7524919125585545, 1.1804156983562801, 1.1778760010857563, 1.0298326204965722, 0.9385140198493713, 1.0207128740101707, 0.929812017339553, 0.8640701821512353, 0.8130222503970004, 0.7736044339144612, 0.7280851293348032, 0.7061941984091654, 0.6826155087046274, 1.642189790371069, 1.511740343745529, 1.3334033796706957, 1.1637276240936476, 2.2882576030527964, 1.895449319429852, 1.5552551751392194, 1.3067312409844312, 1.222859749246748, 0.9319757616478574, 0.7801162715542103, 0.6613743325062905, 0.561952429248885, 0.43456344744462927, 0.3748431739999157, 0.36656867778116903 ], "peak_layer": 21, "peak_normalized": 2.2882576030527964, "population_cv": 0.4268460433011898, "rest_mean": 7.697251674943362e-05, "spike_contrast": 1.8814776376552176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014482206897810103, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.761372060282156e-05, 7.037409523036331e-05, 6.971538823563606e-05, 6.589873373741284e-05, 0.00010337373532820493, 0.00010315132385585457, 9.018657146953046e-05, 8.21894354885444e-05, 8.9387918706052e-05, 8.142736623995006e-05, 7.567009015474468e-05, 7.119961810531095e-05, 6.774764187866822e-05, 6.376133387675509e-05, 6.184425728861243e-05, 5.977937689749524e-05, 0.0001438131439499557, 0.0001323891628999263, 0.00011677147995214909, 0.00010191229375777766, 0.00020039189257659018, 0.00016599209629930556, 0.00013619993114843965, 0.00011443569383118302, 0.00010709073103498667, 8.161685400409624e-05, 6.831790960859507e-05, 5.79192019358743e-05, 4.92124272568617e-05, 3.8056463381508365e-05, 3.282651960034855e-05, 3.2101888791657984e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.07233077567070723, "first_to_last_ratio": 1.3967980239296907, "imbalance_abs_log_ratio": 0.33418249138977923, "last_quartile_mean": 0.051783274626359344, "mean": 0.07766150805400684, "normalized": [ 0.7720752144189068, 0.8035957101529103, 0.7960739453756324, 0.7524919172501093, 1.18041575335872, 1.177876023895764, 1.029832619521225, 0.9385140290506606, 1.020712886859698, 0.9298120043210532, 0.8640702058481301, 0.8130222480432568, 0.7736043957649873, 0.7280851679574961, 0.7061942119475418, 0.6826155393692982, 1.6421898574274953, 1.51174026509585, 1.333403346061099, 1.1637275827667504, 2.288257583675922, 1.8954492273341264, 1.5552551855736059, 1.3067312459654492, 1.2228597336334397, 0.9319757582706437, 0.7801162839815503, 0.6613743139447658, 0.5619524390301038, 0.4345634639005874, 0.3748431737420666, 0.366568666461154 ], "peak_layer": 21, "peak_normalized": 2.288257583675922, "population_cv": 0.4268460335100709, "rest_mean": 0.06825999867309022, "spike_contrast": 1.8814776033915557, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12842965871095657, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.05996052548289299, 0.0624084547162056, 0.06182430312037468, 0.05843965709209442, 0.0916728675365448, 0.09147562831640244, 0.07997835427522659, 0.07288641482591629, 0.07927010208368301, 0.07221060246229172, 0.0671049952507019, 0.06314053386449814, 0.06007928401231766, 0.056544192135334015, 0.05484410747885704, 0.05301295220851898, 0.12753494083881378, 0.1174040287733078, 0.10355411469936371, 0.0903768390417099, 0.17770953476428986, 0.1472034454345703, 0.12078346312046051, 0.10148271918296814, 0.09496913105249405, 0.07237864285707474, 0.060585007071495056, 0.051363326609134674, 0.0436420738697052, 0.03374885395169258, 0.02911088615655899, 0.02846827544271946 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.100046171766735e-06, "first_to_last_ratio": 0.7824902199643116, "imbalance_abs_log_ratio": 0.24527385509409916, "last_quartile_mean": 3.961769863281006e-06, "mean": 4.656072917441634e-06, "normalized": [ 0.7040746582663436, 0.6663619356936908, 0.6544453195323515, 0.6681125781418429, 0.6743036280537461, 0.6669931612309391, 0.6466275673249758, 0.6455375483541481, 0.766065733134754, 0.7325311317531802, 0.713184552491017, 0.7279868059896252, 0.6487420214900163, 0.6567739586893749, 0.6698952555545961, 0.731415182041189, 1.5336327145648883, 1.46016794598778, 1.3941212250673451, 1.3066200632810194, 2.5346433175887912, 2.2579967860286922, 1.9597071198477958, 1.7730016814327627, 1.6211494054956397, 1.2094753028064131, 1.0219643022327503, 0.8604311185936954, 0.6846742153365659, 0.5134546057775913, 0.45040862343156707, 0.44550053478491114 ], "peak_layer": 21, "peak_normalized": 2.5346433175887912, "population_cv": 0.5361380970640629, "rest_mean": 3.768573976601517e-06, "spike_contrast": 2.50719960829824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.448567197978264e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.278222948210896e-06, 3.1026297619973775e-06, 3.0471451282210182e-06, 3.110780880888342e-06, 3.139606860713684e-06, 3.1055687941261567e-06, 3.0107451038929867e-06, 3.0056698960834183e-06, 3.566857913028798e-06, 3.4107183637388516e-06, 3.3206392799911555e-06, 3.3895596516231308e-06, 3.020590156666003e-06, 3.0579874419345288e-06, 3.119081156910397e-06, 3.405522420507623e-06, 7.140705747588072e-06, 6.798648428230081e-06, 6.491130079666618e-06, 6.083718290028628e-06, 1.1801484106399585e-05, 1.0513397683098447e-05, 9.124539246840868e-06, 8.255225111497566e-06, 7.548189842054853e-06, 5.631405201711459e-06, 4.7583403102180455e-06, 4.006230028608115e-06, 3.187893071299186e-06, 2.3906820842967136e-06, 2.097135393341887e-06, 2.0742829747177893e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.958586689303047e-05, "first_to_last_ratio": 1.5974368171613162, "imbalance_abs_log_ratio": 0.46840035541466, "last_quartile_mean": 3.730092248588335e-05, "mean": 6.105391446453723e-05, "normalized": [ 0.841921083374255, 0.8656847679595863, 0.8583386074530126, 0.8192653470430583, 1.1963759198218682, 1.175200963423096, 1.067618556331515, 0.9832339424318312, 1.0816814073351606, 0.984735398821402, 0.9088666847085389, 0.8483372799998844, 0.8441732679586115, 0.788240769992009, 0.7677328825781261, 0.7384848720107285, 1.592019271873236, 1.469773741071107, 1.2839883253170383, 1.1290190536252611, 2.340759428283372, 1.8604406810329672, 1.4645591028269243, 1.2019442625318217, 1.1579689537744897, 0.8703227892706042, 0.7241054984123771, 0.6218678661137057, 0.4685539623066287, 0.375211854314897, 0.3379493629430677, 0.3316240950598181 ], "peak_layer": 21, "peak_normalized": 2.340759428283372, "population_cv": 0.4213561912347865, "rest_mean": 5.421209425549023e-05, "spike_contrast": 1.8077099757766004, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.799974359339103e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 5.140257781022228e-05, 5.2853443776257336e-05, 5.240493192104623e-05, 5.001935642212629e-05, 7.30434330762364e-05, 7.175061909947544e-05, 6.518229201901704e-05, 6.003028101986274e-05, 6.604088412132114e-05, 6.012195080984384e-05, 5.5489868827862665e-05, 5.179431173019111e-05, 5.154008249519393e-05, 4.812518454855308e-05, 4.687309774453752e-05, 4.508739220909774e-05, 9.71990084508434e-05, 8.973544026957825e-05, 7.839251338737085e-05, 6.893103272886947e-05, 0.00014291252591647208, 0.00011358718620613217, 8.941706619225442e-05, 7.338340219575912e-05, 7.069853745633736e-05, 5.3136613132664934e-05, 4.420947516337037e-05, 3.796746750595048e-05, 2.860705353668891e-05, 2.2908152459422126e-05, 2.0633131498470902e-05, 2.0246949134161696e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.3941625335719436e-05, "first_to_last_ratio": 1.9900849942152503, "imbalance_abs_log_ratio": 0.6881773484852566, "last_quartile_mean": 2.2080275698499463e-05, "mean": 4.1715027776945135e-05, "normalized": [ 0.9392326235815728, 0.967237934106481, 0.9603764045369728, 0.9164866155582375, 1.2577449343388738, 1.232917566027963, 1.1238782174568975, 1.0291368181202014, 1.1532132229581178, 1.0411575328021039, 0.9502116988067248, 0.8875233874351255, 0.938078657297939, 0.8686978298587725, 0.846151620562448, 0.8110684117898503, 1.5702254340289712, 1.447397095355284, 1.2505015110000162, 1.0952804003622336, 2.3931370003923895, 1.7749904370362708, 1.2920876407146809, 1.0187689124624395, 1.0659664997461884, 0.8050885780431608, 0.6754901061622123, 0.6027145753846552, 0.3199047099172539, 0.266999929991275, 0.2517914183349422, 0.24654227582974492 ], "peak_layer": 21, "peak_normalized": 2.3931370003923895, "population_cv": 0.43040686028555775, "rest_mean": 3.7783076651168005e-05, "spike_contrast": 1.666025359377284, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.294756385614165e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.918011498171836e-05, 4.034835728816688e-05, 4.006212839158252e-05, 3.8231264625210315e-05, 5.2466864872258157e-05, 5.143119051354006e-05, 4.688261105911806e-05, 4.293047095416114e-05, 4.810632162843831e-05, 4.343191540101543e-05, 3.963810740970075e-05, 3.70230627595447e-05, 3.9131977246142924e-05, 3.6237754102330655e-05, 3.5297238355269656e-05, 3.383374132681638e-05, 6.550199759658426e-05, 6.037821003701538e-05, 5.216470526647754e-05, 4.568965232465416e-05, 9.98297764454037e-05, 7.404377538478002e-05, 5.3899471822660416e-05, 4.2497973481658846e-05, 4.446682214620523e-05, 3.3584292395971715e-05, 2.8178088541608304e-05, 2.5142255253740586e-05, 1.3344833860173821e-05, 1.1137909496028442e-05, 1.0503486009838525e-05, 1.0284517884429079e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015718752911197953, "first_to_last_ratio": 1.257989643214417, "imbalance_abs_log_ratio": 0.22951492550542246, "last_quartile_mean": 0.0001249513697985094, "mean": 0.0001807773865039053, "normalized": [ 0.7253332250389443, 0.7487212712295568, 0.7410605164905312, 0.7105636788603447, 1.0746269026158506, 1.051977329709352, 0.9748117817952817, 0.9289757254955786, 1.003966745690328, 0.9387623076773945, 0.8790880476775305, 0.8198592543393499, 0.76275878383003, 0.7097319103357375, 0.6897181090899124, 0.6787200572192026, 1.6053206581539505, 1.5070819773591158, 1.3319581978459651, 1.179141640369754, 2.363943782302499, 1.985198270384524, 1.6420557999668541, 1.4171107521036512, 1.2788263571440766, 0.9695939295226049, 0.8072936975349638, 0.6913356827376413, 0.5679249559528513, 0.4460061883008719, 0.39061284360066933, 0.37791961962508147 ], "peak_layer": 21, "peak_normalized": 2.363943782302499, "population_cv": 0.44512103777248124, "rest_mean": 0.00015609032642613683, "spike_contrast": 2.012216375705279, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000314087510923855, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.00013112384476698935, 0.00013535187463276088, 0.00013396698341239244, 0.00012845384480897337, 0.0001942682429216802, 0.00019017371232621372, 0.00017622392624616623, 0.00016793780378066003, 0.00018149448442272842, 0.00016970699653029442, 0.00015891923976596445, 0.00014821201330050826, 0.0001378895394736901, 0.00012830347986891866, 0.0001246854371856898, 0.0001226972381118685, 0.00029020567308180034, 0.0002724463411141187, 0.0002407879219390452, 0.00021316214406397194, 0.00042734757880680263, 0.00035887895501218736, 0.0002968465560115874, 0.0002561815781518817, 0.00023118288663681597, 0.00017528065654914826, 0.00014594044478144497, 0.00012497785792220384, 0.00010266798926750198, 8.062783308560029e-05, 7.061396900098771e-05, 6.831932114437222e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08626964688301086, 0.27676042914390564, 0.41335058212280273, 0.43042075634002686, 0.1360139101743698, 0.15445689857006073, 0.1670466512441635, 0.17834730446338654, 0.05234000086784363, 0.08269349485635757, 0.11548293381929398, 0.16410532593727112, 0.062187470495700836, 0.0988159254193306, 0.13199056684970856, 0.17199495434761047, 0.04149825870990753, 0.06434677541255951, 0.08728200942277908, 0.11647984385490417, 0.038355328142642975, 0.06353354454040527, 0.09196189045906067, 0.1169828325510025, 0.027557097375392914, 0.06579430401325226, 0.09665275365114212, 0.12799666821956635, 0.019497457891702652, 0.07176875323057175, 0.14131556451320648, 0.22250843048095703 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.0937054614478257e-05, "first_to_last_ratio": 1.1688073264187404, "imbalance_abs_log_ratio": 0.15598384976011784, "last_quartile_mean": 1.791317879451526e-05, "mean": 2.4262353377935142e-05, "normalized": [ 0.7126199170124646, 0.7437761981288968, 0.7319360705844501, 0.6938843441336064, 1.08510119201235, 1.0981398918115777, 0.9606406303851301, 0.877454527863037, 1.0089327386637434, 0.8940806225700821, 0.830291493091461, 0.7704822352676253, 0.7457403811840673, 0.695661023016061, 0.6629855157677164, 0.6359363332477325, 1.598966161144301, 1.4898256875513118, 1.3481551517699815, 1.1823992139387995, 2.2648560798112616, 2.007845512240135, 1.6690690306890001, 1.384726637362728, 1.3105208676356837, 1.0955359004357528, 0.840135724559922, 0.6990787570111011, 0.728702616979868, 0.5032438811110385, 0.39261683155295724, 0.3366588314661564 ], "peak_layer": 21, "peak_normalized": 2.2648560798112616, "population_cv": 0.43870656486100834, "rest_mean": 2.0994108227670142e-05, "spike_contrast": 1.9963161442660275, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1910877189366144e-05, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.7289836250711232e-05, 1.8045760953100398e-05, 1.7758491594577208e-05, 1.6835267160786316e-05, 2.6327108571422286e-05, 2.6643458113539964e-05, 2.3307402443606406e-05, 2.128911182808224e-05, 2.4479082640027627e-05, 2.1692500013159588e-05, 2.014482561207842e-05, 1.8693712263484485e-05, 1.8093416656483896e-05, 1.6878373571671546e-05, 1.6085588868008927e-05, 1.542931204312481e-05, 3.8794682041043416e-05, 3.6146677302895114e-05, 3.270941670052707e-05, 2.8687787562375888e-05, 5.495073855854571e-05, 4.871505734627135e-05, 4.04955426347442e-05, 3.3596727007534355e-05, 3.179632039973512e-05, 2.6580279154586606e-05, 2.0383669834700413e-05, 1.696129584161099e-05, 1.768004040059168e-05, 1.2209880878799595e-05, 9.525808309263084e-06, 8.168135536834598e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 8.209185853047529e-05, "first_to_last_ratio": 1.1473585117140146, "imbalance_abs_log_ratio": 0.13746235401177884, "last_quartile_mean": 7.154856803026632e-05, "mean": 9.638741801154538e-05, "normalized": [ 0.701459601059617, 0.7307291493400122, 0.7197289413331719, 0.6836538994288254, 1.0739127818136336, 1.080318121017968, 0.9514366180464413, 0.8722525427024255, 1.0096073400424832, 0.8908667057242938, 0.8272733289673383, 0.7688963021177191, 0.7419276606217693, 0.6953759857067083, 0.6657591236204922, 0.6349357593481342, 1.6060262927227642, 1.5018600810151583, 1.362811062623623, 1.194289437021534, 2.265619973525555, 2.010448271283051, 1.6756913579512287, 1.3967040891245648, 1.314898968363092, 1.100096740194715, 0.8429885804297745, 0.7025081852971004, 0.7359520665807775, 0.5075723672109625, 0.3944131736981434, 0.33998549206692247 ], "peak_layer": 21, "peak_normalized": 2.265619973525555, "population_cv": 0.44132695283622186, "rest_mean": 8.330956363890975e-05, "spike_contrast": 2.0046657830023333, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016700783162377776, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.761187978554517e-05, 7.043309597065672e-05, 6.937281432328746e-05, 6.58956341794692e-05, 0.00010351168020861223, 0.00010412907431600615, 9.170651901513338e-05, 8.407417044509202e-05, 9.731344471219927e-05, 8.58683415572159e-05, 7.973874016897753e-05, 7.411192927975208e-05, 7.151249155867845e-05, 6.70254958095029e-05, 6.417080294340849e-05, 6.119981844676659e-05, 0.0001548007276142016, 0.00014476041542366147, 0.0001313578395638615, 0.0001151144751929678, 0.00021837725944351405, 0.0001937819179147482, 0.0001615155633771792, 0.00013462470087688416, 0.00012673971650656313, 0.00010603548435028642, 8.125349268084392e-05, 6.771295011276379e-05, 7.093651947798207e-05, 4.8923589929472655e-05, 3.80164674425032e-05, 3.27703237417154e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.07279988285154104, "first_to_last_ratio": 1.147358527887077, "imbalance_abs_log_ratio": 0.13746236810768883, "last_quartile_mean": 0.06344998627901077, "mean": 0.08547732653096318, "normalized": [ 0.7014596295781764, 0.7307291764187523, 0.7197289409168642, 0.6836539457276586, 1.0739127543434965, 1.0803181180576646, 0.9514366494990735, 0.8722525785288534, 1.0096072992260057, 0.8908667122670901, 0.8272733032016506, 0.7688963353200844, 0.7419276673674994, 0.6953759896953364, 0.6657591362695728, 0.6349357531030216, 1.6060263641500938, 1.5018600149298467, 1.3628109924626157, 1.1942894172476932, 2.2656200131573585, 2.010448125243662, 1.6756912972456086, 1.3967041753456768, 1.3148989834510798, 1.10009676829185, 0.8429886102104137, 0.7025081738287764, 0.7359520241866415, 0.5075723834629702, 0.3944131950658124, 0.3399854721990999 ], "peak_layer": 21, "peak_normalized": 2.2656200131573585, "population_cv": 0.44132694138829454, "rest_mean": 0.07387975465368342, "spike_contrast": 2.0046657621228294, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14810421466827392, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.059958893805742264, 0.062460776418447495, 0.06152050569653511, 0.058436911553144455, 0.0917951911687851, 0.09234270453453064, 0.08132626116275787, 0.0745578184723854, 0.08629853278398514, 0.07614890486001968, 0.070713110268116, 0.06572320312261581, 0.06341799348592758, 0.059438880532979965, 0.05690731108188629, 0.05427261069417, 0.13727883994579315, 0.12837497889995575, 0.11648944020271301, 0.10208466649055481, 0.19365914165973663, 0.17184773087501526, 0.14323361217975616, 0.11938653886318207, 0.1123940497636795, 0.09403333067893982, 0.07205641269683838, 0.06004852056503296, 0.06290721148252487, 0.043385930359363556, 0.033713385462760925, 0.029061049222946167 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.1205255481836502e-06, "first_to_last_ratio": 0.6479717368101099, "imbalance_abs_log_ratio": 0.4339081996148503, "last_quartile_mean": 4.8158358936234436e-06, "mean": 5.188215141060937e-06, "normalized": [ 0.6392337912530196, 0.5994569684566948, 0.58506750248592, 0.6009069195183814, 0.6062570789746512, 0.60249307876806, 0.5896500227462853, 0.5886480511239991, 0.7270810166784253, 0.6920427312432549, 0.6762719442051294, 0.6775019374497203, 0.6128892519841179, 0.6178328466527419, 0.6247970380766965, 0.662130571719626, 1.514294792953365, 1.477854106612226, 1.4442983342347777, 1.3462892680196958, 2.490378094515379, 2.363646805945382, 2.0879100705160045, 1.7472602290519488, 1.6663413461773389, 1.404838629343315, 1.0886161574596454, 0.888478735318369, 0.8979335469262955, 0.6001271407896346, 0.4703453234739583, 0.40912666732594116 ], "peak_layer": 21, "peak_normalized": 2.490378094515379, "population_cv": 0.5663116287539589, "rest_mean": 4.159116037448661e-06, "spike_contrast": 2.583565883667621, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0745350300567224e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.3164824344567023e-06, 3.1101117201615125e-06, 3.035456074940157e-06, 3.117634378213552e-06, 3.1453921565116616e-06, 3.1258637136488687e-06, 3.0592311759392032e-06, 3.0540327315975446e-06, 3.7722527395089855e-06, 3.5904665764974197e-06, 3.508644340399769e-06, 3.515025809974759e-06, 3.1798012969375122e-06, 3.205449729648535e-06, 3.2415814530395437e-06, 3.435275857555098e-06, 7.856487172830384e-06, 7.667425052204635e-06, 7.493330485885963e-06, 6.984838364587631e-06, 1.2920617336931173e-05, 1.2263108146726154e-05, 1.0832526641024742e-05, 9.06516197574092e-06, 8.645337402413134e-06, 7.28860504750628e-06, 5.647974830935709e-06, 4.6096188270894345e-06, 4.658672423829557e-06, 3.1135887184063904e-06, 2.4402527287747944e-06, 2.122637170032249e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.989002966089174e-05, "first_to_last_ratio": 1.332672418740526, "imbalance_abs_log_ratio": 0.2871862636142851, "last_quartile_mean": 4.4939798272025655e-05, "mean": 6.640715508865469e-05, "normalized": [ 0.7737597503853951, 0.7960652246538951, 0.7852109175536554, 0.752979652136534, 1.1002854313277015, 1.0894969185314263, 0.9958159097459445, 0.9212749625084216, 1.0695780627909957, 0.9470852845577153, 0.8771168497054519, 0.8087458852142236, 0.8138331380491499, 0.758274367943245, 0.7300990118419535, 0.6932937549003495, 1.563540331611732, 1.463522142564802, 1.3107588349120618, 1.1481313121287318, 2.3272535492615467, 1.9917532184740583, 1.5842339276075041, 1.284041417038419, 1.2647947479005226, 1.026614217953409, 0.784658048181791, 0.6569384849801976, 0.6050998048035263, 0.4243821080145578, 0.34340906908209917, 0.3079536636389831 ], "peak_layer": 21, "peak_normalized": 2.3272535492615467, "population_cv": 0.432791558992841, "rest_mean": 5.791668829462853e-05, "spike_contrast": 1.938226771623043, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011225567577639594, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 5.138318374520168e-05, 5.2864426834275946e-05, 5.2143623179290444e-05, 5.000323653803207e-05, 7.3066825279966e-05, 7.235039083752781e-05, 6.61293015582487e-05, 6.117924931459129e-05, 7.102763629518449e-05, 6.289323937380686e-05, 5.824683466926217e-05, 5.370651342673227e-05, 5.404434341471642e-05, 5.035484355175868e-05, 4.8483798309462145e-05, 4.603966590366326e-05, 0.00010383026528870687, 9.7188341896981e-05, 8.704376523382962e-05, 7.62441341066733e-05, 0.0001545462873764336, 0.0001322666648775339, 0.00010520446812734008, 8.526953752152622e-05, 8.399142097914591e-05, 6.817452958784997e-05, 5.2106908697169274e-05, 4.362541585578583e-05, 4.0182956581702456e-05, 2.8182008463772945e-05, 2.2804819309385493e-05, 2.045032670139335e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.413846727402415e-05, "first_to_last_ratio": 1.6934944304798134, "imbalance_abs_log_ratio": 0.5267941045072542, "last_quartile_mean": 2.6063544396492944e-05, "mean": 4.4781047250808115e-05, "normalized": [ 0.8740739154412697, 0.9009770250766957, 0.891219784649098, 0.852663960345562, 1.1726907227894316, 1.1562245824076949, 1.0625075493400142, 0.9748474712247537, 1.1459503353768645, 1.011393273941681, 0.9321659114804322, 0.8550679923070413, 0.9071127008648373, 0.8438105066025905, 0.8153663695855309, 0.7686703721764182, 1.5553617227891403, 1.437584791446802, 1.2800134621565402, 1.1209839695222465, 2.385210126491363, 1.925331940523583, 1.4016057137556681, 1.0729916373601829, 1.178493073924358, 0.9257430549345239, 0.725497883628933, 0.6192210996384143, 0.4345528238561406, 0.29242012088135483, 0.2468894193108289, 0.23335668617000463 ], "peak_layer": 21, "peak_normalized": 2.385210126491363, "population_cv": 0.4349311871982457, "rest_mean": 3.986569293003952e-05, "spike_contrast": 1.7891062550480503, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.132396058295853e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.9141945308074355e-05, 4.034669473185204e-05, 3.990975528722629e-05, 3.818318509729579e-05, 5.251431866781786e-05, 5.177694765734486e-05, 4.758020077133551e-05, 4.36546906712465e-05, 5.131685611559078e-05, 4.5291249989531934e-05, 4.174336572759785e-05, 3.829084016615525e-05, 4.062145671923645e-05, 3.7786718166898936e-05, 3.651295992312953e-05, 3.442186425672844e-05, 6.965072680031881e-05, 6.437655247282237e-05, 5.732034333050251e-05, 5.019883610657416e-05, 0.00010681220737751573, 8.621838060207665e-05, 6.27653716946952e-05, 4.804968921234831e-05, 5.277415402815677e-05, 4.145574348513037e-05, 3.2488555007148534e-05, 2.772936932160519e-05, 1.9459730538073927e-05, 1.3094879250274971e-05, 1.1055966751882806e-05, 1.0449956789670978e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015772300503158476, "first_to_last_ratio": 1.024373509074145, "imbalance_abs_log_ratio": 0.02408121506561912, "last_quartile_mean": 0.00015397021070384653, "mean": 0.0001989226409477851, "normalized": [ 0.6591656486528042, 0.6829581384685631, 0.6699127961863572, 0.6472837418583299, 0.9802229283334125, 0.9691875554589603, 0.8879032543387678, 0.8464550603982911, 0.9884366921662843, 0.8930886037902819, 0.8342806121981681, 0.779517431659101, 0.7202640807077498, 0.675490688069157, 0.6529307639307617, 0.6324384542682998, 1.559547816640635, 1.474378252555848, 1.3651032473841427, 1.1832746816977402, 2.3416687863410064, 2.09906600553135, 1.7771997929015884, 1.4880606167336252, 1.412667012460112, 1.1576278202661294, 0.8849667940826665, 0.7258688199763643, 0.7424188068351703, 0.5131968143231243, 0.4021086961884269, 0.3533095855967808 ], "peak_layer": 21, "peak_normalized": 2.3416687863410064, "population_cv": 0.4654332615698335, "rest_mean": 0.0001685783755857739, "spike_contrast": 2.152005989155233, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003627816739026457, "top_five_layers": [ 21, 22, 23, 17, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.00013112297165207565, 0.00013585583656094968, 0.00013326082262210548, 0.0001287593913730234, 0.00019498853362165391, 0.00019279334810562432, 0.00017662406025920063, 0.00016837907605804503, 0.00019662243721541017, 0.00017765554366633296, 0.00016595730266999453, 0.00015506366617046297, 0.00014327683311421424, 0.00013437039160635322, 0.00012988271191716194, 0.00012580632755998522, 0.00031022937037050724, 0.00029328721575438976, 0.0002715499431360513, 0.00023538012464996427, 0.0004658109392039478, 0.0004175517533440143, 0.00035352527629584074, 0.0002960089477710426, 0.00028101145289838314, 0.00023027838324196637, 0.00017603993183001876, 0.0001443917426513508, 0.0001476839097449556, 0.00010208646563114598, 7.998852379387245e-05, 7.028127583907917e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16424456238746643, 0.29588937759399414, 0.42337876558303833, 0.45154979825019836, 0.13891471922397614, 0.1555756777524948, 0.16914354264736176, 0.1874145120382309, 0.06211879849433899, 0.09842939674854279, 0.137006014585495, 0.20118924975395203, 0.08399304747581482, 0.12138351052999496, 0.1587214469909668, 0.2071460783481598, 0.0581100657582283, 0.08358953893184662, 0.10947012156248093, 0.1472771316766739, 0.05584336072206497, 0.08318652212619781, 0.11238116025924683, 0.1379220336675644, 0.05995604768395424, 0.09392217546701431, 0.1259489357471466, 0.17086055874824524, 0.06700707972049713, 0.13971327245235443, 0.2211977243423462, 0.32473331689834595 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.0823702925554244e-05, "first_to_last_ratio": 1.4242510302785252, "imbalance_abs_log_ratio": 0.35364608276130804, "last_quartile_mean": 1.4620809452026151e-05, "mean": 2.2054440051988422e-05, "normalized": [ 0.7841928807641645, 0.8183408510588258, 0.809986731151413, 0.7637938668182616, 1.192276091281044, 1.1966830153099992, 1.039215154291241, 0.9490753634193776, 1.0189872806238243, 0.9331490903420598, 0.8658120301757168, 0.81301306861575, 0.7774052511262896, 0.7266654164061733, 0.7030327996851689, 0.6829495498627286, 1.6322510246428148, 1.499422250137046, 1.3169815172043178, 1.1572342326306069, 2.2864238933994527, 1.8918479809103845, 1.543069371898747, 1.294657154506133, 1.2171810844243176, 0.9287685590307506, 0.7771677166674748, 0.6578712720326597, 0.5567773486562734, 0.43254113576640124, 0.3708732313504376, 0.3623537858101448 ], "peak_layer": 21, "peak_normalized": 2.2864238933994527, "population_cv": 0.4245664907667088, "rest_mean": 1.9413478454495175e-05, "spike_contrast": 1.8706401721656991, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6315632678451947e-05, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 1.729493487800937e-05, 1.804804924177006e-05, 1.78638038050849e-05, 1.684504604781978e-05, 2.629498158057686e-05, 2.639217382238712e-05, 2.2919308321434073e-05, 2.093132570735179e-05, 2.2473193894256838e-05, 2.0580080672516488e-05, 1.9094999515800737e-05, 1.7930547983269207e-05, 1.7145237507065758e-05, 1.6026198863983154e-05, 1.5504994735238142e-05, 1.5062069905980024e-05, 3.5998382372781634e-05, 3.306891812826507e-05, 2.9045289920759387e-05, 2.5522153009660542e-05, 5.0425798690412194e-05, 4.172364788246341e-05, 3.403153095860034e-05, 2.855293860193342e-05, 2.6844247258850373e-05, 2.048347050731536e-05, 1.7139998817583546e-05, 1.450898253096966e-05, 1.2279412658244837e-05, 9.539452548779082e-06, 8.179401447705459e-06, 7.991509846760891e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 8.156212697940646e-05, "first_to_last_ratio": 1.3967870279562358, "imbalance_abs_log_ratio": 0.33417461908716395, "last_quartile_mean": 5.839267214469146e-05, "mean": 8.757388354752038e-05, "normalized": [ 0.7720675832605176, 0.803585946279625, 0.7960780094481321, 0.752473219070419, 1.180467409539882, 1.1778079020041263, 1.029829529518624, 0.9385078067458692, 1.0206984705622164, 0.9298229900567075, 0.8640759923745102, 0.8130195958778703, 0.7736070401530517, 0.7280910766467854, 0.7061969571118035, 0.6826252956632183, 1.642168884788434, 1.5117445082936212, 1.3334176736203447, 1.1637251407789435, 2.2882673690329693, 1.8954517085610634, 1.5552436162725376, 1.306771834154562, 1.2228641636923565, 0.9319785952743014, 0.7801153976292595, 0.661371584878177, 0.561946672486053, 0.4345672285647456, 0.37483694087850516, 0.36657385678076787 ], "peak_layer": 21, "peak_normalized": 2.2882673690329693, "population_cv": 0.42684708327825066, "rest_mean": 7.697225831819836e-05, "spike_contrast": 1.8814916302334765, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001448226597858593, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 6.761295662727207e-05, 7.037314207991585e-05, 6.971564289415255e-05, 6.589700205950066e-05, 0.00010337811545468867, 0.00010314521205145866, 9.018617129186168e-05, 8.218877337640151e-05, 8.93865289981477e-05, 8.14282102510333e-05, 7.567049033241346e-05, 7.11992834112607e-05, 6.774777284590527e-05, 6.376176315825433e-05, 6.184441008372232e-05, 5.9780148149002343e-05, 0.00014381110668182373, 0.00013238933752290905, 0.0001167725640698336, 0.00010191192995989695, 0.0002003924601012841, 0.0001659920671954751, 0.0001361987233394757, 0.00011443908442743123, 0.00010709096386563033, 8.16169849713333e-05, 6.831773498561233e-05, 5.791887815576047e-05, 4.921185245621018e-05, 3.8056739867897704e-05, 3.282592660980299e-05, 3.210229624528438e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.07233011117205024, "first_to_last_ratio": 1.3967870631582553, "imbalance_abs_log_ratio": 0.3341746442893017, "last_quartile_mean": 0.051783205242827535, "mean": 0.07766139676095918, "normalized": [ 0.7720675426341356, 0.8035859729306402, 0.7960780122636479, 0.7524732326455996, 1.1804674279966365, 1.1778079658527474, 1.0298294903648106, 0.9385077949910019, 1.020698424099846, 0.9298230264026498, 0.8640760490713295, 0.8130195756795249, 0.773607039370292, 0.7280910561484961, 0.7061969508238138, 0.6826252958058289, 1.64216899408666, 1.511744542110794, 1.3334176327397504, 1.1637251251728324, 2.288267386572505, 1.8954517517435265, 1.5552435035136007, 1.3067718770227776, 1.2228641722871045, 0.9319785328940787, 0.7801153392913037, 0.6613715681663117, 0.5619466726726858, 0.43456725256605405, 0.37483694737332934, 0.36657384470568466 ], "peak_layer": 21, "peak_normalized": 2.288267386572505, "population_cv": 0.4268470914878033, "rest_mean": 0.0682597693607763, "spike_contrast": 1.8814916300573643, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12843018472194673, "top_five_layers": [ 21, 22, 17, 23, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.05995984375476837, 0.062407609075307846, 0.061824530363082886, 0.058438122272491455, 0.0916767492890358, 0.09147021174430847, 0.07997799664735794, 0.07288582623004913, 0.07926886528730392, 0.072211354970932, 0.06710535287857056, 0.06314023584127426, 0.06007940322160721, 0.05654456838965416, 0.054844241589307785, 0.0530136339366436, 0.12753313779830933, 0.11740419268608093, 0.10355507582426071, 0.09037651866674423, 0.17771004140377045, 0.14720343053340912, 0.12078238278627396, 0.1014857292175293, 0.09496933966875076, 0.07237875461578369, 0.06058484688401222, 0.05136303976178169, 0.04364156350493431, 0.03374909982085228, 0.029110360890626907, 0.02846863679587841 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.1000530213987076e-06, "first_to_last_ratio": 0.7825005490137864, "imbalance_abs_log_ratio": 0.2452606549530124, "last_quartile_mean": 3.961726321222159e-06, "mean": 4.656059921614997e-06, "normalized": [ 0.7040313055728622, 0.6663442620495864, 0.6544029514954961, 0.6681637652166335, 0.6743674803950376, 0.6670078174062475, 0.6466284443239834, 0.6455370061269436, 0.7660325155889827, 0.7325145706427032, 0.7131598309497137, 0.7280186266136806, 0.64870437442493, 0.656793469733594, 0.6699169519151534, 0.7314456448845376, 1.53365232903732, 1.4601427212016398, 1.3940973786195396, 1.306558663485338, 2.534675981169938, 2.2580403975857246, 1.9597194264610087, 1.773041790596292, 1.6211455309545355, 1.2094494759638503, 1.0219317012739941, 0.8604275624593173, 0.6846808632651675, 0.5134649266877133, 0.4503974279453127, 0.4455048059532228 ], "peak_layer": 21, "peak_normalized": 2.534675981169938, "population_cv": 0.5361438958058841, "rest_mean": 3.768541933755774e-06, "spike_contrast": 2.5072447705625382, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.4486570560548e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.2780119454400847e-06, 3.1025388125272e-06, 3.046939355044742e-06, 3.1110105283005396e-06, 3.1398953979078215e-06, 3.105628366029123e-06, 3.0107407837931532e-06, 3.005658982146997e-06, 3.566693294487777e-06, 3.4106317343685078e-06, 3.320514906590688e-06, 3.389698349565151e-06, 3.020406438736245e-06, 3.058069751205039e-06, 3.119173470622627e-06, 3.405654751986731e-06, 7.140777142922161e-06, 6.798512004024815e-06, 6.4910009314189665e-06, 6.083415428292938e-06, 1.1801603250205517e-05, 1.0513571396586485e-05, 9.12457107915543e-06, 8.255388820543885e-06, 7.548150733782677e-06, 5.631269232253544e-06, 4.758175236929674e-06, 4.006202289019711e-06, 3.187915126545704e-06, 2.3907234663056443e-06, 2.0970774130546488e-06, 2.074297071885667e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.95857286498358e-05, "first_to_last_ratio": 1.5974327409797537, "imbalance_abs_log_ratio": 0.4683978037101298, "last_quartile_mean": 3.7300931126083015e-05, "mean": 6.105390025368251e-05, "normalized": [ 0.8419145460820371, 0.8656854461464137, 0.8583401181387939, 0.8192663123570689, 1.1963719080724353, 1.1751919414919585, 1.0676149913033914, 0.9832376272965914, 1.0816749854357006, 0.984736879340849, 0.9088680283961561, 0.8483339022774173, 0.8441772779737997, 0.7882426814662962, 0.7677338358974833, 0.7384876656990601, 1.5920129687595903, 1.4697744406924866, 1.283993033566668, 1.1290203889690138, 2.340760926498109, 1.860435632123778, 1.4645539617726704, 1.2019638482713701, 1.15797649283655, 0.8703237664687024, 0.7241030451553558, 0.6218659849233957, 0.4685510324632258, 0.37521378882561096, 0.3379462835275562, 0.3316262577704645 ], "peak_layer": 21, "peak_normalized": 2.340760926498109, "population_cv": 0.4213559298637173, "rest_mean": 5.4212039955288036e-05, "spike_contrast": 1.8077155175462722, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.79999458650127e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 5.140216671861708e-05, 5.285347288008779e-05, 5.240501195657998e-05, 5.0019403715850785e-05, 7.304317114176229e-05, 7.175005157478154e-05, 6.518205918837339e-05, 6.003049202263355e-05, 6.604047666769475e-05, 6.012202720739879e-05, 5.548993794946e-05, 5.1794093451462686e-05, 5.154031532583758e-05, 4.8125290049938485e-05, 4.687314503826201e-05, 4.5087552280165255e-05, 9.7198600997217e-05, 8.973546209745109e-05, 7.839278259780258e-05, 6.8931098212488e-05, 0.000142912584124133, 0.00011358685151208192, 8.941673149820417e-05, 7.338458090089262e-05, 7.069898128975183e-05, 5.3136660426389426e-05, 4.420931509230286e-05, 3.796734381467104e-05, 2.8606867999769747e-05, 2.2908265236765146e-05, 2.0632938685594127e-05, 2.0247076463419944e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.39446339441929e-05, "first_to_last_ratio": 1.9902838030955072, "imbalance_abs_log_ratio": 0.6882772431887607, "last_quartile_mean": 2.207958175404201e-05, "mean": 4.171556903997953e-05, "normalized": [ 0.9393219483952423, 0.9671725353750462, 0.9606035942825071, 0.9165296658237206, 1.25754251069206, 1.233040231347365, 1.1241638961033311, 1.029104366181827, 1.1530467776467852, 1.041181174787795, 0.9503458810810488, 0.8874956508445306, 0.9380795670187313, 0.868712023499227, 0.8459898570744635, 0.811097306638917, 1.57041471098841, 1.447423664007262, 1.250399123009104, 1.0954141829132344, 2.392471764387743, 1.7753028127291355, 1.2921083756975622, 1.0187283101804303, 1.065894151409315, 0.8051193818729687, 0.6754751497722067, 0.6026093861097862, 0.3198852539229254, 0.2669904264118248, 0.2517857094641488, 0.2465506103313451 ], "peak_layer": 21, "peak_normalized": 2.392471764387743, "population_cv": 0.4303681174959756, "rest_mean": 3.778425454786177e-05, "spike_contrast": 1.6658967617763285, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.294466729741543e-05, "top_five_layers": [ 21, 22, 17, 18, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.9184349589049816e-05, 4.034615267300978e-05, 4.007212555734441e-05, 3.823355655185878e-05, 5.245910142548382e-05, 5.1436974899843335e-05, 4.689513662015088e-05, 4.2929674236802384e-05, 4.810000245925039e-05, 4.343346517998725e-05, 3.9644219214096665e-05, 3.702238609548658e-05, 3.913252294296399e-05, 3.6238816392142326e-05, 3.5290948289912194e-05, 3.3835385693237185e-05, 6.551074329763651e-05, 6.0380101785995066e-05, 5.2161110943416134e-05, 4.5695825974689797e-05, 9.980332106351852e-05, 7.40577670512721e-05, 5.3901036153547466e-05, 4.249683115631342e-05, 4.446438106242567e-05, 3.358601315994747e-05, 2.8177830245113e-05, 2.513819345040247e-05, 1.3344195394893177e-05, 1.113765756599605e-05, 1.0503384146431927e-05, 1.0284999007126316e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015719397015345749, "first_to_last_ratio": 1.2580343238755638, "imbalance_abs_log_ratio": 0.22955044238581457, "last_quartile_mean": 0.00012495205191953573, "mean": 0.00018078252287523355, "normalized": [ 0.7253227592072276, 0.7485993006094415, 0.7410050906139998, 0.7104508417729404, 1.0745241672934553, 1.0524252536641314, 0.9748384190355198, 0.9289919933888637, 1.0038768041568893, 0.9389629508381758, 0.8789427325469985, 0.8198897306139241, 0.7627520038160613, 0.7096499260969147, 0.6897553416925117, 0.6787601781063402, 1.6052869610938005, 1.5071148227224969, 1.3317635520239959, 1.1792135859285877, 2.364258320942869, 1.9856700690456726, 1.6414783683894763, 1.4170804705838413, 1.2788477375082987, 0.9694880608375548, 0.8072605380605845, 0.691415531221962, 0.5679201295495192, 0.44599504581333255, 0.3904354851204105, 0.3780238277042025 ], "peak_layer": 21, "peak_normalized": 2.364258320942869, "population_cv": 0.4451626458968529, "rest_mean": 0.0001560934222113617, "spike_contrast": 2.0122799667677387, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00031410366646014156, "top_five_layers": [ 21, 22, 23, 17, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.00013112567830830812, 0.0001353336701868102, 0.00013396076974458992, 0.00012843709555454552, 0.00019425518985372037, 0.0001902600924950093, 0.00017623374878894538, 0.00016794551629573107, 0.00018148338131140918, 0.00016974809113889933, 0.00015889748465269804, 0.0001482217339798808, 0.00013789223157800734, 0.00012829230399802327, 0.00012469571083784103, 0.00012270797742530704, 0.00029020782676525414, 0.00027246001991443336, 0.00024075957480818033, 0.0002131812070729211, 0.00042741658398881555, 0.0003589744446799159, 0.00029675060068257153, 0.00025618338258937, 0.0002311933203600347, 0.00017526649753563106, 0.00014593859668821096, 0.0001249958440894261, 0.00010267003381159157, 8.062810957198963e-05, 7.058391202008352e-05, 6.834010127931833e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.062413111329078674, 0.07552354782819748, 0.14653873443603516, 0.2105521410703659, 0.2186809629201889, 0.14272794127464294, 0.1287408471107483, 0.13253600895404816, 0.13371841609477997, 0.09585829079151154, 0.09883332997560501, 0.10020580142736435, 0.11237040907144547, 0.08538161963224411, 0.08663177490234375, 0.10358677804470062, 0.10874549299478531, 0.06467291712760925, 0.07278503477573395, 0.08346924930810928, 0.09857872873544693, 0.06093711405992508, 0.06834813207387924, 0.0680762529373169, 0.07369214296340942, 0.05633483827114105, 0.05850831791758537, 0.06479237228631973, 0.06705968081951141, 0.061747051775455475, 0.0654391348361969, 0.07205834239721298 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0717411839777924e-05, "first_to_last_ratio": 1.3606399895734356, "imbalance_abs_log_ratio": 0.3079551696283513, "last_quartile_mean": 7.876743239876305e-06, "mean": 1.4356073158694471e-05, "normalized": [ 0.6895052247736269, 0.5576007830925034, 0.38530829240192566, 0.5472592297661263, 0.37729668968423546, 0.9131958145391527, 1.3435179101111983, 1.1586522070597358, 1.5436961923733754, 1.144684350345656, 0.8911764746394834, 0.9261744836030649, 0.5468539631209283, 0.8612404659079811, 0.9670035934822814, 0.889769793290417, 0.9785434614239343, 1.8290819610081128, 1.5223690430889059, 1.8851307751720068, 1.8951492301881587, 2.5144698996677146, 2.058593895387357, 1.1843681717440393, 1.0244158825153251, 0.9136429572594602, 0.5473929037729057, 0.6024761608101054, 0.3781148252878133, 0.4421926007844957, 0.24629038211477544, 0.23483238158319855 ], "peak_layer": 22, "peak_normalized": 2.5144698996677146, "population_cv": 0.5646113035804416, "rest_mean": 1.2400990229892799e-05, "spike_contrast": 2.008994484502458, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4913520974223502e-05, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 9.898587450152263e-06, 8.004957635421306e-06, 5.531514034373686e-06, 7.856493539293297e-06, 5.41649887964013e-06, 1.3109905921737663e-05, 1.9287641407572664e-05, 1.6633695850032382e-05, 2.216141547251027e-05, 1.643317227717489e-05, 1.279379466723185e-05, 1.3296228644321673e-05, 7.850675501686055e-06, 1.2364031135803089e-05, 1.3882374332752079e-05, 1.2773600246873684e-05, 1.4048041521164123e-05, 2.6258434445480816e-05, 2.185524135711603e-05, 2.7063075322075747e-05, 2.7206900995224714e-05, 3.6097913834964857e-05, 2.9553324566222727e-05, 1.7002876120386645e-05, 1.4706589354318567e-05, 1.3116325135342777e-05, 7.858412573114038e-06, 8.649191840959247e-06, 5.428244094218826e-06, 6.348149327095598e-06, 3.5357627439225325e-06, 3.3712708500388544e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.013745751763054e-05, "first_to_last_ratio": 1.3002217470857882, "imbalance_abs_log_ratio": 0.26253482460265976, "last_quartile_mean": 3.0869701731717214e-05, "mean": 5.617674656832605e-05, "normalized": [ 0.473654064493233, 0.5256902444154287, 0.3705068590661575, 0.5387329076786941, 0.37665460925127686, 0.9021763170314637, 1.3591261829588805, 1.169340915014312, 1.4504557484696843, 1.2215794360380687, 0.9103877586499164, 0.9321134746924985, 0.5414129154069731, 0.8525397685215896, 0.9808579608967221, 0.9142526712062526, 1.0619023556310665, 1.8034129735144708, 1.5888135435441055, 1.899281403844036, 1.9024910148779852, 2.5497535502321647, 2.089500431013523, 1.18928036938163, 1.0430326636758747, 0.9143825787913804, 0.5551399248346642, 0.5581364123079678, 0.38887965127025986, 0.4518317596354874, 0.24874412658654727, 0.2359354070676856 ], "peak_layer": 22, "peak_normalized": 2.5497535502321647, "population_cv": 0.5766137085977164, "rest_mean": 4.832436503353098e-05, "spike_contrast": 2.0399565889343343, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.857960685621947e-05, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 2.6608344342093915e-05, 2.9531567633966915e-05, 2.0813869923586026e-05, 3.0264262022683397e-05, 2.1159230527700856e-05, 5.0681330321822315e-05, 7.635128713445738e-05, 6.568976823473349e-05, 8.148188499035314e-05, 6.862435839138925e-05, 5.114262239658274e-05, 5.236310244072229e-05, 3.041481613763608e-05, 4.7892910515656695e-05, 5.510140908882022e-05, 5.135974060976878e-05, 5.9654219512594864e-05, 0.00010130987357115373, 8.925437578000128e-05, 0.00010669545008568093, 0.00010687575559131801, 0.00014323685900308192, 0.00011738133616745472, 6.680990190943703e-05, 5.859418160980567e-05, 5.1367038395255804e-05, 3.1185954867396504e-05, 3.1354287784779444e-05, 2.18459936149884e-05, 2.538243825256359e-05, 1.397363575961208e-05, 1.3254083569336217e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03559429990127683, "first_to_last_ratio": 1.3002217714306916, "imbalance_abs_log_ratio": 0.26253484332631455, "last_quartile_mean": 0.027375560603104532, "mean": 0.04981810107710771, "normalized": [ 0.4736540568825577, 0.5256902401367205, 0.3705068792758801, 0.5387329172865728, 0.3766546277313775, 0.9021763186028472, 1.3591262384871188, 1.1693409622603343, 1.4504557122308583, 1.2215794195883674, 0.9103877486678197, 0.9321134797583928, 0.5414129179041209, 0.8525397564320455, 0.9808579422244733, 0.9142526416467389, 1.0619023214513024, 1.8034129787908137, 1.5888135716490905, 1.8992814677966294, 1.9024909329539326, 2.549753462436128, 2.0895004555311574, 1.1892804001617727, 1.0430326554170677, 0.9143826055433697, 0.5551398871242083, 0.5581364223645632, 0.38887964652236373, 0.4518317875925886, 0.24874413075622867, 0.23593541479255786 ], "peak_layer": 22, "peak_normalized": 2.549753462436128, "population_cv": 0.576613701889011, "rest_mean": 0.04285453077129744, "spike_contrast": 2.0399565496359413, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0874213807284832, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 0.023596545681357384, 0.026188889518380165, 0.01845794916152954, 0.026838650926947594, 0.01876421831548214, 0.04494471102952957, 0.06770908832550049, 0.05825434625148773, 0.07225894927978516, 0.06085676699876785, 0.04535378888249397, 0.0464361235499382, 0.02697216346859932, 0.04247191175818443, 0.04886448010802269, 0.04554633051156998, 0.05290195718407631, 0.08984261006116867, 0.07915167510509491, 0.09461859613656998, 0.09477848559617996, 0.1270238757133484, 0.10409494489431381, 0.0592476911842823, 0.051961906254291534, 0.045552805066108704, 0.027656015008687973, 0.027805296704173088, 0.019373245537281036, 0.02250940166413784, 0.012391960248351097, 0.011753854341804981 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.9162920753833532e-06, "first_to_last_ratio": 0.5172826991978068, "imbalance_abs_log_ratio": 0.6591657469591501, "last_quartile_mean": 5.637714309614239e-06, "mean": 5.31696678152116e-06, "normalized": [ 2.0689578843648566, 0.7607510573443436, 0.20852787293451516, 0.2204829621221139, 0.15259464703358938, 0.24520248807427852, 0.36182592024390203, 0.3695605650129213, 0.49013872876325654, 0.6336599249096766, 0.533666626864933, 0.7065919516088963, 0.4409106734215795, 0.6204696462782906, 0.6094215771745697, 0.6245105255232631, 0.9007337205411854, 1.4762581910365227, 1.5659025652557783, 1.5082894705927152, 1.6990202709747757, 2.997351654903827, 2.2171258799622118, 2.105442927603072, 1.6398303896765056, 1.6495827585670528, 1.159798242314629, 1.3298749851714036, 0.9845526396022054, 0.7951372535426381, 0.492335162782499, 0.4314908357979929 ], "peak_layer": 22, "peak_normalized": 2.997351654903827, "population_cv": 0.6973153224686441, "rest_mean": 4.202615000563297e-06, "spike_contrast": 2.697003269910379, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.133446639869362e-05, "top_five_layers": [ 22, 23, 24, 1, 21 ], "top_five_spike_overlap": 4 }, "values": [ 1.1000580343534239e-05, 4.044888100906974e-06, 1.1087357734140824e-06, 1.1723005854946678e-06, 8.113406693155412e-07, 1.3037334838372772e-06, 1.9238163986301515e-06, 1.9649412479338935e-06, 2.6060513391712448e-06, 3.3691487715259427e-06, 2.8374877274472965e-06, 3.756925934794708e-06, 2.344307404200663e-06, 3.299016498203855e-06, 3.2402742817794206e-06, 3.3205017189175123e-06, 4.789171271113446e-06, 7.84921576268971e-06, 8.325851922563743e-06, 8.019525012059603e-06, 9.033634341903962e-06, 1.593681918166112e-05, 1.1788384654209949e-05, 1.1194570106454194e-05, 8.71892370923888e-06, 8.77077673067106e-06, 6.166608727653511e-06, 7.070901119732298e-06, 5.2348336794239e-06, 4.227718363836175e-06, 2.61772970588936e-06, 2.2942224404687295e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6059751689899713e-05, "first_to_last_ratio": 1.3136640344119184, "imbalance_abs_log_ratio": 0.27282020579643, "last_quartile_mean": 1.983745539746451e-05, "mean": 3.78474055935385e-05, "normalized": [ 0.4589265841635195, 0.5077348811615244, 0.3979439201623806, 0.5340807477186122, 0.3758546364010748, 0.8955899993665016, 1.2433535731478862, 1.0948988680288454, 1.2525996711021592, 1.2070776025128485, 0.9384156458392604, 0.9009485279466896, 0.5787782313540217, 0.9014501901107522, 1.0394159362329198, 0.9428128555264055, 1.0906052781563889, 1.901538177289176, 1.645145075817088, 1.8092152293363273, 1.9503048533412066, 2.7077203289997303, 2.1756312348621445, 1.2568133833623651, 1.064835664650375, 0.9479102198797764, 0.5517473493893477, 0.45064300663075274, 0.3856334920760448, 0.38842166290606656, 0.2110006297428454, 0.19295254278496235 ], "peak_layer": 22, "peak_normalized": 2.7077203289997303, "population_cv": 0.6018204393551218, "rest_mean": 3.202268221179515e-05, "spike_contrast": 2.164119526172185, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.930091185495258e-05, "top_five_layers": [ 22, 23, 21, 18, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.7369180568493903e-05, 1.9216447981307283e-05, 1.5061144949868321e-05, 2.021357067860663e-05, 1.4225122868083417e-05, 3.389575795154087e-05, 4.7057706979103386e-05, 4.143908154219389e-05, 4.7407647798536345e-05, 4.5684755605179816e-05, 3.5516597563400865e-05, 3.4098564356099814e-05, 2.1905254470766522e-05, 3.4117550967494026e-05, 3.933919651899487e-05, 3.568302054191008e-05, 4.127658030483872e-05, 7.196828664746135e-05, 6.226447294466197e-05, 6.847410259069875e-05, 7.381397881545126e-05, 0.00010248018952552229, 8.23419977677986e-05, 4.7567125875502825e-05, 4.0301267290487885e-05, 3.5875942558050156e-05, 2.0882205717498437e-05, 1.7055668649845757e-05, 1.4595227185054682e-05, 1.4700752217322588e-05, 7.985826414369512e-06, 7.30275314708706e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7762501443030487e-05, "first_to_last_ratio": 1.3504079548741488, "imbalance_abs_log_ratio": 0.3004067355961661, "last_quartile_mean": 1.315343365604349e-05, "mean": 2.547085748005884e-05, "normalized": [ 0.4678386796616822, 0.5235377308817779, 0.4345577931245205, 0.5165838835160134, 0.4023318777827698, 0.9134377309740589, 1.2276035467300503, 1.0930340705226127, 1.0712949866382682, 1.183459795509594, 0.9400000096186449, 0.8234489915172668, 0.6264659982400982, 0.9214742940814803, 1.1003532746415148, 0.9455412775127405, 1.1042840733917474, 2.0049913068036904, 1.659392669807193, 1.6624482120354451, 1.950545580645667, 2.815676235928119, 2.2178876005698376, 1.2625215400320522, 1.0704850029998623, 1.0091870523571758, 0.5486605781901173, 0.4093310376700093, 0.39616480433203954, 0.33219377199499983, 0.19307701107731026, 0.17218958121164055 ], "peak_layer": 22, "peak_normalized": 2.815676235928119, "population_cv": 0.6142210141608739, "rest_mean": 2.1398240985743338e-05, "spike_contrast": 2.218078886997531, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.746298654936254e-05, "top_five_layers": [ 22, 23, 18, 21, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.1916252333321609e-05, 1.3334954928723164e-05, 1.1068559615523554e-05, 1.3157834473531693e-05, 1.0247737918689381e-05, 2.326604226254858e-05, 3.126811498077586e-05, 2.7840515031130053e-05, 2.728680192376487e-05, 3.014373578480445e-05, 2.3942606276250444e-05, 2.0973951905034482e-05, 1.5956626157276332e-05, 2.347074041608721e-05, 2.8026941436110064e-05, 2.4083747121039778e-05, 2.8127062250860035e-05, 5.1068847824353725e-05, 4.226615419611335e-05, 4.234398147673346e-05, 4.96820684929844e-05, 7.171768811531365e-05, 5.6491498980904e-05, 3.21575062116608e-05, 2.7266170945949852e-05, 2.5704859581310302e-05, 1.3974855392007157e-05, 1.0426012522657402e-05, 1.0090657269756775e-05, 8.461260222247802e-06, 4.917837031825911e-06, 4.385816282592714e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.240166451083496e-05, "first_to_last_ratio": 1.2489384660588578, "imbalance_abs_log_ratio": 0.22229396336363683, "last_quartile_mean": 5.797056178380444e-05, "mean": 0.0001109465127342446, "normalized": [ 0.42709475759614274, 0.4935186328821003, 0.35950105255417747, 0.5439303076283766, 0.3371404368406644, 0.8629008054703304, 1.152554863343196, 1.0440127885102133, 1.3928110595765304, 1.2466636602290324, 0.9528986062756032, 0.9925221789536534, 0.5398804000340625, 0.8579108303439223, 1.0148735477787485, 0.9293336653216302, 1.0803774560382864, 1.8519808466056424, 1.6712809075368884, 1.9834945531761723, 1.984928279946317, 2.6376973498062735, 2.185821980542675, 1.2767982858711757, 1.0594658474140766, 0.9033078060139075, 0.5489550405623127, 0.4646003337611295, 0.361861238846865, 0.4230227492827023, 0.22047516727493316, 0.19838456398225895 ], "peak_layer": 22, "peak_normalized": 2.6376973498062735, "population_cv": 0.610312229060459, "rest_mean": 9.391535294761761e-05, "spike_contrast": 2.1606134589645687, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002029147755820304, "top_five_layers": [ 22, 23, 21, 20, 18 ], "top_five_spike_overlap": 3 }, "values": [ 4.738467396236956e-05, 5.475417128764093e-05, 3.988538810517639e-05, 6.034717080183327e-05, 3.7404555769171566e-05, 9.573583520250395e-05, 0.00012787194282282144, 0.00011582957813516259, 0.00015452752995770425, 0.00013831298565492034, 0.00010572077735560015, 0.0001101168745663017, 5.989784767734818e-05, 9.518221486359835e-05, 0.00011259668099228293, 0.00010310632933396846, 0.00011986411118414253, 0.00020547081658151, 0.00018542278849054128, 0.00022006180370226502, 0.0002202208706876263, 0.000292643322609365, 0.00024250932619906962, 0.00014165631728246808, 0.00011754404113162309, 0.00010021885100286454, 6.090464739827439e-05, 5.154578684596345e-05, 4.0147242543753237e-05, 4.693289884016849e-05, 2.4460950953653082e-05, 2.2010075554135256e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05931045487523079, 0.07934419065713882, 0.214645653963089, 0.09514089673757553, 0.12821291387081146, 0.1321839988231659, 0.12568901479244232, 0.1335759311914444, 0.07788944244384766, 0.08352623134851456, 0.07934392243623734, 0.09343641996383667, 0.07063088566064835, 0.06870647519826889, 0.07622843235731125, 0.08666495233774185, 0.057347409427165985, 0.05777324363589287, 0.06351501494646072, 0.07469023764133453, 0.051319729536771774, 0.05102822557091713, 0.05902284011244774, 0.06656526774168015, 0.050236333161592484, 0.05184732377529144, 0.061248794198036194, 0.06608261168003082, 0.048912134021520615, 0.062179163098335266, 0.07131517678499222, 0.06802479177713394 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.254158395473496e-06, "first_to_last_ratio": 0.18864041654826996, "imbalance_abs_log_ratio": 1.6679126340303851, "last_quartile_mean": 2.7852771381731145e-05, "mean": 2.0484676667820167e-05, "normalized": [ 0.5568020220803858, 0.21809281295181746, 0.09241273564310358, 0.28328072581730895, 0.22196567434120804, 0.13250710048221157, 0.17764328005295377, 0.3692327060422689, 0.6392853696302555, 0.4685551353916866, 0.4862788629304224, 0.46993740268393913, 0.5775584528492825, 0.5275503726871256, 0.4833126690762866, 0.4805557266835708, 1.3911160583811022, 1.414259456489031, 1.4619454359332327, 1.4084306067413008, 2.477707616941058, 2.525352305517634, 2.2589272888438794, 1.9997852708388022, 2.0849570534601547, 1.8699232722164576, 1.4787615635622833, 1.3487564232655025, 1.428990706094454, 1.0620321056863733, 0.8416254501901257, 0.762458336494781 ], "peak_layer": 22, "peak_normalized": 2.525352305517634, "population_cv": 0.7251468208972824, "rest_mean": 1.566946546652751e-05, "spike_contrast": 2.966713654279006, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.648681715480052e-05, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1405909390305169e-05, 4.467560756893363e-06, 1.8930450096377172e-06, 5.802914074592991e-06, 4.546895070234314e-06, 2.7143651095684618e-06, 3.6389651540957857e-06, 7.563612598460168e-06, 1.3095554095343687e-05, 9.598200449545402e-06, 9.961265277524944e-06, 9.626515748095699e-06, 1.1831098163384013e-05, 1.0806698810483795e-05, 9.900503755488899e-06, 9.844028681982309e-06, 2.8496562663349323e-05, 2.8970647690584883e-05, 2.9947479561087675e-05, 2.8851245588157326e-05, 5.07550394104328e-05, 5.1731025450862944e-05, 4.6273395128082484e-05, 4.096495467820205e-05, 4.270967110642232e-05, 3.830477362498641e-05, 3.0291952498373576e-05, 2.7628839234239422e-05, 2.927241257566493e-05, 2.1755384295829572e-05, 1.7240425222553313e-05, 1.561871249577962e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.0563243310789403e-05, "first_to_last_ratio": 0.184584542859635, "imbalance_abs_log_ratio": 1.6896476935739415, "last_quartile_mean": 0.00011140284550492652, "mean": 8.183052003118974e-05, "normalized": [ 0.5514618447323059, 0.2169408826973503, 0.08472485593965659, 0.28295099514537675, 0.21989392642956848, 0.1303640614627248, 0.17290402482659964, 0.35108448581118906, 0.6388397786640535, 0.4693907558756991, 0.4878178928069798, 0.4684568375678178, 0.5735758405807379, 0.5222431877258841, 0.48692296371012395, 0.47894342377930604, 1.3828038952387727, 1.4137992065669855, 1.4773604231197706, 1.4101154596735193, 2.482879371711871, 2.526943137473127, 2.259311590178407, 2.019190865749238, 2.085961378955964, 1.8764944434843154, 1.4765035496268348, 1.350239590483524, 1.4291199659133378, 1.0658000032658883, 0.838995628185017, 0.7679657326180543 ], "peak_layer": 22, "peak_normalized": 2.526943137473127, "population_cv": 0.7282772544392188, "rest_mean": 6.251158794367695e-05, "spike_contrast": 2.977891930556664, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001861527533037588, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.51264095318038e-05, 1.7752385247149505e-05, 6.933079021109734e-06, 2.3154027076088823e-05, 1.7994034351431765e-05, 1.0667758942872752e-05, 1.4148826267046388e-05, 2.8729426048812456e-05, 5.2276591304689646e-05, 3.841048965114169e-05, 3.9918391848914325e-05, 3.833406663034111e-05, 4.6936009312048554e-05, 4.2735431634355336e-05, 3.984515933552757e-05, 3.91921894333791e-05, 0.00011315556184854358, 0.00011569192429305986, 0.00012089317169738933, 0.00011539048136910424, 0.00020317531016189605, 0.00020678107102867216, 0.00018488064233679324, 0.00016523143858648837, 0.00017069530440494418, 0.0001535545161459595, 0.00012082305329386145, 0.00011049080785596743, 0.00011694562999764457, 8.721496851649135e-05, 6.865544855827466e-05, 6.2843035266269e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.018235690018627793, "first_to_last_ratio": 0.1845845477639835, "imbalance_abs_log_ratio": 1.6896476670042821, "last_quartile_mean": 0.09879315597936511, "mean": 0.07256812266132329, "normalized": [ 0.5514618483157399, 0.21694088816023477, 0.08472485792509094, 0.28295099765712173, 0.21989393857080072, 0.13036405951384217, 0.17290402763813106, 0.3510845056878191, 0.6388397993688198, 0.4693907637084327, 0.48781790094052196, 0.46845687595408214, 0.5735758195754074, 0.5222432048140645, 0.48692297651101485, 0.47894340052024936, 1.3828038812071648, 1.413799283122749, 1.4773604270157898, 1.4101154778648226, 2.482879404994159, 2.526942972591032, 2.259311440497952, 2.0191909931789223, 2.0859612981908873, 1.8764944638324905, 1.4765035418797068, 1.350239576611916, 1.4291200320637618, 1.065800015673522, 0.8389956311315239, 0.7679656952822267 ], "peak_layer": 22, "peak_normalized": 2.526942972591032, "population_cv": 0.7282772403307106, "rest_mean": 0.0554359013180214, "spike_contrast": 2.977891835258891, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1650821179151535, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.04001855105161667, 0.015742992982268333, 0.006148323882371187, 0.02053322270512581, 0.015957290306687355, 0.009460275061428547, 0.012547320686280727, 0.025477543473243713, 0.04635940492153168, 0.03406280651688576, 0.03540002927184105, 0.03399503603577614, 0.0416233204305172, 0.03789820894598961, 0.03533508628606796, 0.034756023436784744, 0.10034748166799545, 0.10259675979614258, 0.10720927268266678, 0.10232943296432495, 0.18017789721488953, 0.1833755075931549, 0.1639539897441864, 0.14652889966964722, 0.15137429535388947, 0.13617368042469025, 0.10714709013700485, 0.09798435121774673, 0.10370855778455734, 0.07734310626983643, 0.06088433787226677, 0.05572982877492905 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0810431092522776e-06, "first_to_last_ratio": 0.14665609099981078, "imbalance_abs_log_ratio": 1.9196649501572336, "last_quartile_mean": 7.371279991730262e-06, "mean": 5.266313571894443e-06, "normalized": [ 0.5313084313996962, 0.2061221485022935, 0.06868365811035125, 0.2784811311506817, 0.146950226496972, 0.10534296524196422, 0.12013039774147215, 0.1851819388065788, 0.48851680128093, 0.4019870994251106, 0.45293299696953215, 0.4442480210738259, 0.5787785597712432, 0.5090619802943529, 0.49010374567770754, 0.468585442827116, 1.3525598419019165, 1.4672711047348668, 1.4843408168011953, 1.4896119798089702, 2.508208189766812, 2.524495568813508, 2.388845240425297, 2.1106199865726754, 2.1011836342122523, 1.9283517633380012, 1.5593850323316485, 1.452268444304737, 1.4451925616379866, 1.080259681295224, 0.833132510373883, 0.7978580989111985 ], "peak_layer": 22, "peak_normalized": 2.524495568813508, "population_cv": 0.7672111824742712, "rest_mean": 3.972487093051313e-06, "spike_contrast": 3.084459752954336, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2252976557647343e-05, "top_five_layers": [ 22, 21, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.7980368031421676e-06, 1.0855038681256701e-06, 3.6170968087390065e-07, 1.4665689604953513e-06, 7.738859721939662e-07, 5.547690875573608e-07, 6.326443440229923e-07, 9.752261576068122e-07, 2.5726826606842224e-06, 2.1169901174289407e-06, 2.385287189099472e-06, 2.3395493826683378e-06, 3.048029384444817e-06, 2.6808800157596124e-06, 2.581040007498814e-06, 2.4677178771526087e-06, 7.1230042522074655e-06, 7.727109732513782e-06, 7.817004188837018e-06, 7.844763786124531e-06, 1.3209010830905754e-05, 1.3294785276229959e-05, 1.2580408110807184e-05, 1.1115186680399347e-05, 1.1065491889894474e-05, 1.0155305062653497e-05, 8.212210559577215e-06, 7.648101018276066e-06, 7.610837201355025e-06, 5.688986220775405e-06, 4.387537046568468e-06, 4.201770934741944e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.3072703836769506e-05, "first_to_last_ratio": 0.1965017087277629, "imbalance_abs_log_ratio": 1.6270841518950423, "last_quartile_mean": 6.65271763864439e-05, "mean": 5.1720684069778144e-05, "normalized": [ 0.5903214985473635, 0.21317222278028428, 0.07861556264200854, 0.2924267620633159, 0.20614520303339837, 0.10942131635502071, 0.17754724840214117, 0.3543967343826241, 0.6616591248095567, 0.5148181180855445, 0.5447776590149115, 0.5218723062533341, 0.647120485093606, 0.5989696729166056, 0.5592579173992114, 0.5462615999231498, 1.3998993635309975, 1.4086638785298187, 1.4578445936683029, 1.4102355321134408, 2.6382627874666973, 2.6317584037312556, 2.2233659362975056, 1.922962332424717, 2.1034577149820564, 1.8142894928770414, 1.4101545016386676, 1.2891070706275354, 1.273679909247231, 0.9355476159814242, 0.756254888856092, 0.7077325463251402 ], "peak_layer": 21, "peak_normalized": 2.6382627874666973, "population_cv": 0.7263037787473766, "rest_mean": 3.923146602944514e-05, "spike_contrast": 3.0374205592556516, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011916246148757637, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.0531831725966185e-05, 1.1025413186871447e-05, 4.066050678375177e-06, 1.5124512174224947e-05, 1.0661970918590669e-05, 5.659345333697274e-06, 9.182865142065566e-06, 1.8329641534364782e-05, 3.4221462556160986e-05, 2.6626745238900185e-05, 2.8176273190183565e-05, 2.6991592676495202e-05, 3.346951416460797e-05, 3.097912122029811e-05, 2.8925202059326693e-05, 2.825302362907678e-05, 7.240375271067023e-05, 7.285705942194909e-05, 7.540071965195239e-05, 7.293834642041475e-05, 0.0001364527561236173, 0.0001361163449473679, 0.00011499400716274977, 9.945692727342248e-05, 0.00010879227193072438, 9.383629367221147e-05, 7.293415546882898e-05, 6.667349953204393e-05, 6.587559619219974e-05, 4.838716267840937e-05, 3.911402018275112e-05, 3.66044114343822e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.603489931147124e-06, "first_to_last_ratio": 0.24312184321824193, "imbalance_abs_log_ratio": 1.4141925488860427, "last_quartile_mean": 3.538756459420256e-05, "mean": 3.111726851301455e-05, "normalized": [ 0.6644756490180851, 0.23325159254807834, 0.07539325824570257, 0.31873771579525234, 0.21217410245742602, 0.11960013716276979, 0.20155955466263742, 0.38669620325415055, 0.7284622302362078, 0.5930713003867844, 0.6319198952855785, 0.6040604923468361, 0.7580808064483824, 0.7072262953008145, 0.6749843915485383, 0.6486223376959581, 1.4735493318409316, 1.4845067822216826, 1.4896409680595757, 1.476749210387741, 2.817405493456774, 2.7673279203027326, 2.139795377674301, 1.694850102870422, 2.0495665188393017, 1.6765832039465556, 1.3012719394146397, 1.1897383429944555, 1.0088778638486162, 0.720783924666597, 0.5951684074906716, 0.5558686495917995 ], "peak_layer": 21, "peak_normalized": 2.817405493456774, "population_cv": 0.7201026839240032, "rest_mean": 2.366186438646562e-05, "spike_contrast": 3.0165184632367974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.137645079637877e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.067666719085537e-05, 7.258152436406817e-06, 2.3460322609025752e-06, 9.918247087625787e-06, 6.602278517675586e-06, 3.721629582287278e-06, 6.271982783800922e-06, 1.2032929589622654e-05, 2.2667754819849506e-05, 1.845475890149828e-05, 1.9663621060317382e-05, 1.879671253846027e-05, 2.3589404008816928e-05, 2.2006950530339964e-05, 2.1003670553909615e-05, 2.018335544562433e-05, 4.5852830226067454e-05, 4.619379615178332e-05, 4.635355799109675e-05, 4.5952401706017554e-05, 8.76699632499367e-05, 8.611168595962226e-05, 6.658458732999861e-05, 5.273910574032925e-05, 6.377691170200706e-05, 5.2170689741615206e-05, 4.0492028347216547e-05, 3.7021407479187474e-05, 3.139352338621393e-05, 2.242882692371495e-05, 1.8520015146350488e-05, 1.729711402731482e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.8654464106002706e-05, "first_to_last_ratio": 0.16200207727357052, "imbalance_abs_log_ratio": 1.8201461211556016, "last_quartile_mean": 0.0002386047435720684, "mean": 0.00017063225249103198, "normalized": [ 0.5177972227971404, 0.18422958638068, 0.07755682312933251, 0.27342225532372755, 0.19007362166945208, 0.0968768622809422, 0.1517726472113855, 0.32056442710365024, 0.6168627493486994, 0.4509605691569947, 0.4954745315627688, 0.46627480155634954, 0.5531570230685467, 0.5170724935984963, 0.45563757978860675, 0.4789901403808068, 1.3287627572643317, 1.3383475524992587, 1.407284240753668, 1.3213535994248635, 2.4744625690346806, 2.5664095372695273, 2.4114518937040392, 2.1183513814824377, 2.201741632911359, 1.9351067846360057, 1.5669730065860399, 1.4334150862915997, 1.3851211553907088, 1.0454346613928627, 0.8612433454555685, 0.7578174615454693 ], "peak_layer": 22, "peak_normalized": 2.5664095372695273, "population_cv": 0.7569752509959051, "rest_mean": 0.00012783252026969826, "spike_contrast": 3.14279031375294, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00040175080648623406, "top_five_layers": [ 22, 21, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 8.83529064594768e-05, 3.1435509299626574e-05, 1.3233695426606573e-05, 4.6654655307065696e-05, 3.2432690204586834e-05, 1.653031722526066e-05, 2.5897308660205454e-05, 5.469863026519306e-05, 0.00010525668039917946, 7.69484176998958e-05, 8.454393537249416e-05, 7.956151966936886e-05, 9.438642882741988e-05, 8.822924428386614e-05, 7.774646655889228e-05, 8.173116657417268e-05, 0.00022672978229820728, 0.0002283652574988082, 0.00024012807989493012, 0.00022546554100699723, 0.00042222312185913324, 0.0004379122401587665, 0.00041147146839648485, 0.0003614590677898377, 0.000375688134226948, 0.00033019162947312, 0.00026737613370642066, 0.00024458684492856264, 0.00023634634271729738, 0.00017838487110566348, 0.00014695589197799563, 0.00012930810044053942 ] } } } } }, "uniform_value_backward": { "bits_per_byte": 1.8565786854442996, "forward": { "activation_sha256": { "attention_branch_output": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", "mlp_branch_output": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", "post_attention_state": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", "post_mlp_state": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", "pre_attention_input": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", "pre_mlp_input": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f" }, "logits_sha256": "a911233a29684ae3ced35122e9aee12a584fd2d6b7e0272c227ab94eb4e607ec", "loss_nats": 1.2868822813034058, "mixer_summary_sha256": "998a9ce6e3ee4cab0180aeb81cc12c18ccf6b1832b43c8f455133227a7e72a26" }, "loss_nats": 1.2868822813034058, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.21598131954669952, "group": 1, "latest_source_mass": 0.061274416744709015, "layer": 1, "max_source_mass": 0.9387255907058716, "mixer_index": 1, "normalized_entropy": 0.31159517863467795, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9387255907058716, "median": 0.9528161883354187, "p05": 0.8532108068466187, "p95": 0.9815421104431152 }, { "label": "current_group_partial", "mean": 0.061274416744709015, "median": 0.047183819115161896, "p05": 0.018457919359207153, "p95": 0.14678917825222015 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.5377621054649353, "group": 1, "latest_source_mass": 0.2917928099632263, "layer": 2, "max_source_mass": 0.7082071900367737, "mixer_index": 2, "normalized_entropy": 0.7758267227322699, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.7082071900367737, "median": 0.7268136143684387, "p05": 0.38468095660209656, "p95": 0.923936128616333 }, { "label": "current_group_partial", "mean": 0.2917928099632263, "median": 0.27318641543388367, "p05": 0.07606390863656998, "p95": 0.6153190732002258 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.43774232268333435, "group": 1, "latest_source_mass": 0.170285165309906, "layer": 2, "max_source_mass": 0.829714834690094, "mixer_index": 3, "normalized_entropy": 0.6315286781224629, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.829714834690094, "median": 0.85542893409729, "p05": 0.687734842300415, "p95": 0.9260365962982178 }, { "label": "current_group_partial", "mean": 0.170285165309906, "median": 0.14457105100154877, "p05": 0.07396337389945984, "p95": 0.31226515769958496 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6749181151390076, "group": 1, "latest_source_mass": 0.45927006006240845, "layer": 3, "max_source_mass": 0.5407299995422363, "mixer_index": 4, "normalized_entropy": 0.9737010177171727, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5407299995422363, "median": 0.5310744643211365, "p05": 0.4232047200202942, "p95": 0.6916534900665283 }, { "label": "current_group_partial", "mean": 0.45927006006240845, "median": 0.4689255356788635, "p05": 0.3083464205265045, "p95": 0.576795220375061 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6663151383399963, "group": 1, "latest_source_mass": 0.49885618686676025, "layer": 3, "max_source_mass": 0.5011438131332397, "mixer_index": 5, "normalized_entropy": 0.9612895457523564, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5011438131332397, "median": 0.4804786741733551, "p05": 0.3554112911224365, "p95": 0.7207092642784119 }, { "label": "current_group_partial", "mean": 0.49885618686676025, "median": 0.5195213556289673, "p05": 0.27929070591926575, "p95": 0.6445887684822083 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6865031719207764, "group": 1, "latest_source_mass": 0.4691343307495117, "layer": 4, "max_source_mass": 0.5308656692504883, "mixer_index": 6, "normalized_entropy": 0.9904147216846476, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5308656692504883, "median": 0.5278033018112183, "p05": 0.45786383748054504, "p95": 0.6093498468399048 }, { "label": "current_group_partial", "mean": 0.4691343307495117, "median": 0.47219666838645935, "p05": 0.3906501829624176, "p95": 0.5421361327171326 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.4133813679218292, "group": 1, "latest_source_mass": 0.15208598971366882, "layer": 4, "max_source_mass": 0.8479140400886536, "mixer_index": 7, "normalized_entropy": 0.5963832494967191, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.8479140400886536, "median": 0.8514594435691833, "p05": 0.7530128359794617, "p95": 0.9402707815170288 }, { "label": "current_group_partial", "mean": 0.15208598971366882, "median": 0.14854055643081665, "p05": 0.05972917377948761, "p95": 0.24698717892169952 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6850745677947998, "group": 2, "latest_source_mass": 0.4601486325263977, "layer": 5, "max_source_mass": 0.5398513674736023, "mixer_index": 8, "normalized_entropy": 0.9883536815967077, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5398513674736023, "median": 0.5387102365493774, "p05": 0.46027374267578125, "p95": 0.623892605304718 }, { "label": "completed_group_1", "mean": 0.4601486325263977, "median": 0.46128979325294495, "p05": 0.376107394695282, "p95": 0.5397262573242188 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0074119567871094, "group": 2, "latest_source_mass": 0.4627366364002228, "layer": 5, "max_source_mass": 0.4627366364002228, "mixer_index": 9, "normalized_entropy": 0.9169858804405273, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.32744842767715454, "median": 0.30657994747161865, "p05": 0.16577640175819397, "p95": 0.5453717708587646 }, { "label": "completed_group_1", "mean": 0.20981493592262268, "median": 0.2177819311618805, "p05": 0.14342713356018066, "p95": 0.2433384209871292 }, { "label": "current_group_partial", "mean": 0.4627366364002228, "median": 0.4847465753555298, "p05": 0.27109938859939575, "p95": 0.6081984639167786 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0489801168441772, "group": 2, "latest_source_mass": 0.4763939380645752, "layer": 6, "max_source_mass": 0.4763939380645752, "mixer_index": 10, "normalized_entropy": 0.9548228503031734, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28095996379852295, "median": 0.2765299677848816, "p05": 0.23549969494342804, "p95": 0.3424558639526367 }, { "label": "completed_group_1", "mean": 0.24264612793922424, "median": 0.23937124013900757, "p05": 0.21875721216201782, "p95": 0.2814398407936096 }, { "label": "current_group_partial", "mean": 0.4763939380645752, "median": 0.48882943391799927, "p05": 0.38478636741638184, "p95": 0.528400719165802 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.8527575731277466, "group": 2, "latest_source_mass": 0.6454645395278931, "layer": 6, "max_source_mass": 0.6454645395278931, "mixer_index": 11, "normalized_entropy": 0.7762133938639787, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20522089302539825, "median": 0.1591135561466217, "p05": 0.11052663624286652, "p95": 0.47355735301971436 }, { "label": "completed_group_1", "mean": 0.1493145078420639, "median": 0.15203402936458588, "p05": 0.10004506260156631, "p95": 0.18058395385742188 }, { "label": "current_group_partial", "mean": 0.6454645395278931, "median": 0.68653404712677, "p05": 0.4207671880722046, "p95": 0.7333818674087524 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.049942135810852, "group": 2, "latest_source_mass": 0.4395729899406433, "layer": 7, "max_source_mass": 0.4395729899406433, "mixer_index": 12, "normalized_entropy": 0.9556985177033998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35788866877555847, "median": 0.36233147978782654, "p05": 0.30507102608680725, "p95": 0.3985886573791504 }, { "label": "completed_group_1", "mean": 0.20253834128379822, "median": 0.19928529858589172, "p05": 0.180739626288414, "p95": 0.23214691877365112 }, { "label": "current_group_partial", "mean": 0.4395729899406433, "median": 0.43975552916526794, "p05": 0.3959323763847351, "p95": 0.4790603220462799 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9791408181190491, "group": 2, "latest_source_mass": 0.5214842557907104, "layer": 7, "max_source_mass": 0.5214842557907104, "mixer_index": 13, "normalized_entropy": 0.8912523810434521, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.31465935707092285, "median": 0.28992873430252075, "p05": 0.20534247159957886, "p95": 0.507034420967102 }, { "label": "completed_group_1", "mean": 0.1638563871383667, "median": 0.1667816936969757, "p05": 0.11925681680440903, "p95": 0.19515612721443176 }, { "label": "current_group_partial", "mean": 0.5214842557907104, "median": 0.5455081462860107, "p05": 0.35950881242752075, "p95": 0.6097541451454163 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0485806465148926, "group": 2, "latest_source_mass": 0.4457142949104309, "layer": 8, "max_source_mass": 0.4457142949104309, "mixer_index": 14, "normalized_entropy": 0.9544592367395849, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.346283882856369, "median": 0.353412926197052, "p05": 0.26436588168144226, "p95": 0.4084087312221527 }, { "label": "completed_group_1", "mean": 0.20800183713436127, "median": 0.20826736092567444, "p05": 0.18089595437049866, "p95": 0.23509056866168976 }, { "label": "current_group_partial", "mean": 0.4457142949104309, "median": 0.4416901469230652, "p05": 0.3883572816848755, "p95": 0.5136083364486694 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9801348447799683, "group": 2, "latest_source_mass": 0.543371319770813, "layer": 8, "max_source_mass": 0.543371319770813, "mixer_index": 15, "normalized_entropy": 0.8921571831025336, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2748759388923645, "median": 0.2577607035636902, "p05": 0.17991375923156738, "p95": 0.41774845123291016 }, { "label": "completed_group_1", "mean": 0.1817527413368225, "median": 0.17968356609344482, "p05": 0.130966454744339, "p95": 0.23766928911209106 }, { "label": "current_group_partial", "mean": 0.543371319770813, "median": 0.5535831451416016, "p05": 0.4452384412288666, "p95": 0.6142411828041077 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0688585042953491, "group": 3, "latest_source_mass": 0.36256325244903564, "layer": 9, "max_source_mass": 0.4051227569580078, "mixer_index": 16, "normalized_entropy": 0.9729169383233166, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4051227569580078, "median": 0.4008670449256897, "p05": 0.3286888897418976, "p95": 0.48389261960983276 }, { "label": "completed_group_1", "mean": 0.23231396079063416, "median": 0.23353952169418335, "p05": 0.2008797824382782, "p95": 0.2593720257282257 }, { "label": "completed_group_2", "mean": 0.36256325244903564, "median": 0.3638863265514374, "p05": 0.3081618845462799, "p95": 0.4163197875022888 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2915732860565186, "group": 3, "latest_source_mass": 0.36661261320114136, "layer": 9, "max_source_mass": 0.36661261320114136, "mixer_index": 17, "normalized_entropy": 0.931673187369201, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.26621371507644653, "median": 0.23494771122932434, "p05": 0.16815711557865143, "p95": 0.4694361090660095 }, { "label": "completed_group_1", "mean": 0.1108347550034523, "median": 0.11330658197402954, "p05": 0.06620892137289047, "p95": 0.13892118632793427 }, { "label": "completed_group_2", "mean": 0.2563389539718628, "median": 0.26728466153144836, "p05": 0.16010139882564545, "p95": 0.3091832995414734 }, { "label": "current_group_partial", "mean": 0.36661261320114136, "median": 0.36950358748435974, "p05": 0.29210716485977173, "p95": 0.42882072925567627 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3568191528320312, "group": 3, "latest_source_mass": 0.30908840894699097, "layer": 10, "max_source_mass": 0.30908840894699097, "mixer_index": 18, "normalized_entropy": 0.978738131586968, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.294538676738739, "median": 0.2961598038673401, "p05": 0.2467859834432602, "p95": 0.34424129128456116 }, { "label": "completed_group_1", "mean": 0.1710844486951828, "median": 0.17187124490737915, "p05": 0.1508811116218567, "p95": 0.1898101568222046 }, { "label": "completed_group_2", "mean": 0.2252884805202484, "median": 0.22875282168388367, "p05": 0.18443678319454193, "p95": 0.25181543827056885 }, { "label": "current_group_partial", "mean": 0.30908840894699097, "median": 0.3080518841743469, "p05": 0.2719307243824005, "p95": 0.3494151830673218 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.320021152496338, "group": 3, "latest_source_mass": 0.3312946557998657, "layer": 10, "max_source_mass": 0.3312946557998657, "mixer_index": 19, "normalized_entropy": 0.9521939852875004, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30465462803840637, "median": 0.311603307723999, "p05": 0.21955373883247375, "p95": 0.40013614296913147 }, { "label": "completed_group_1", "mean": 0.12911874055862427, "median": 0.12981033325195312, "p05": 0.09804825484752655, "p95": 0.15542283654212952 }, { "label": "completed_group_2", "mean": 0.23493197560310364, "median": 0.24103470146656036, "p05": 0.16810482740402222, "p95": 0.28502923250198364 }, { "label": "current_group_partial", "mean": 0.3312946557998657, "median": 0.325168639421463, "p05": 0.2647673785686493, "p95": 0.41786038875579834 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3578683137893677, "group": 3, "latest_source_mass": 0.30723780393600464, "layer": 11, "max_source_mass": 0.30723780393600464, "mixer_index": 20, "normalized_entropy": 0.9794949412420898, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29558902978897095, "median": 0.2938143014907837, "p05": 0.24335844814777374, "p95": 0.3542291522026062 }, { "label": "completed_group_1", "mean": 0.17812997102737427, "median": 0.17726755142211914, "p05": 0.15839283168315887, "p95": 0.20142368972301483 }, { "label": "completed_group_2", "mean": 0.21904319524765015, "median": 0.2218058705329895, "p05": 0.18076662719249725, "p95": 0.24698719382286072 }, { "label": "current_group_partial", "mean": 0.30723780393600464, "median": 0.30536961555480957, "p05": 0.26194074749946594, "p95": 0.35952210426330566 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.258887767791748, "group": 3, "latest_source_mass": 0.34303945302963257, "layer": 11, "max_source_mass": 0.35744374990463257, "mixer_index": 21, "normalized_entropy": 0.9080955698144659, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.35744374990463257, "median": 0.359974205493927, "p05": 0.17910680174827576, "p95": 0.5207242965698242 }, { "label": "completed_group_1", "mean": 0.12067858874797821, "median": 0.11769672483205795, "p05": 0.08151324093341827, "p95": 0.16476252675056458 }, { "label": "completed_group_2", "mean": 0.17883820831775665, "median": 0.17648206651210785, "p05": 0.0995911955833435, "p95": 0.2739725708961487 }, { "label": "current_group_partial", "mean": 0.34303945302963257, "median": 0.33267199993133545, "p05": 0.22800606489181519, "p95": 0.48691368103027344 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3539760112762451, "group": 3, "latest_source_mass": 0.3175881803035736, "layer": 12, "max_source_mass": 0.3175881803035736, "mixer_index": 22, "normalized_entropy": 0.976687238475429, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2736307978630066, "median": 0.27999642491340637, "p05": 0.18969643115997314, "p95": 0.3464604914188385 }, { "label": "completed_group_1", "mean": 0.1722733974456787, "median": 0.17083553969860077, "p05": 0.1506597101688385, "p95": 0.1988438367843628 }, { "label": "completed_group_2", "mean": 0.2365075945854187, "median": 0.24049004912376404, "p05": 0.19099700450897217, "p95": 0.26942259073257446 }, { "label": "current_group_partial", "mean": 0.3175881803035736, "median": 0.3092706799507141, "p05": 0.268545538187027, "p95": 0.3924510180950165 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3203014135360718, "group": 3, "latest_source_mass": 0.3395293354988098, "layer": 12, "max_source_mass": 0.3395293354988098, "mixer_index": 23, "normalized_entropy": 0.9523961508935896, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28207293152809143, "median": 0.29242122173309326, "p05": 0.1354818046092987, "p95": 0.4195851981639862 }, { "label": "completed_group_1", "mean": 0.15420743823051453, "median": 0.15396761894226074, "p05": 0.11267110705375671, "p95": 0.1951446533203125 }, { "label": "completed_group_2", "mean": 0.22419027984142303, "median": 0.22542357444763184, "p05": 0.1655150204896927, "p95": 0.2747117578983307 }, { "label": "current_group_partial", "mean": 0.3395293354988098, "median": 0.32681703567504883, "p05": 0.2524653375148773, "p95": 0.46652013063430786 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3734662532806396, "group": 4, "latest_source_mass": 0.283242404460907, "layer": 13, "max_source_mass": 0.283242404460907, "mixer_index": 24, "normalized_entropy": 0.9907464762181619, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2766023874282837, "median": 0.282479852437973, "p05": 0.22528076171875, "p95": 0.3097468614578247 }, { "label": "completed_group_1", "mean": 0.21176087856292725, "median": 0.2058793306350708, "p05": 0.18811854720115662, "p95": 0.2617422640323639 }, { "label": "completed_group_2", "mean": 0.22839435935020447, "median": 0.22951346635818481, "p05": 0.20004916191101074, "p95": 0.2551718056201935 }, { "label": "completed_group_3", "mean": 0.283242404460907, "median": 0.2849352955818176, "p05": 0.2205706238746643, "p95": 0.34444743394851685 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5325813293457031, "group": 4, "latest_source_mass": 0.27591708302497864, "layer": 13, "max_source_mass": 0.27591708302497864, "mixer_index": 25, "normalized_entropy": 0.9522463199762954, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17451074719429016, "median": 0.16826045513153076, "p05": 0.08149556815624237, "p95": 0.3126368522644043 }, { "label": "completed_group_1", "mean": 0.10643941164016724, "median": 0.10269765555858612, "p05": 0.08178127557039261, "p95": 0.13947664201259613 }, { "label": "completed_group_2", "mean": 0.1691833734512329, "median": 0.17118802666664124, "p05": 0.12382284551858902, "p95": 0.2069271057844162 }, { "label": "completed_group_3", "mean": 0.27394935488700867, "median": 0.27278125286102295, "p05": 0.2122008204460144, "p95": 0.33945947885513306 }, { "label": "current_group_partial", "mean": 0.27591708302497864, "median": 0.27714359760284424, "p05": 0.21523593366146088, "p95": 0.33170604705810547 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.560360074043274, "group": 4, "latest_source_mass": 0.3255118429660797, "layer": 14, "max_source_mass": 0.3255118429660797, "mixer_index": 26, "normalized_entropy": 0.9695062244951087, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19377154111862183, "median": 0.19472965598106384, "p05": 0.17005598545074463, "p95": 0.22297024726867676 }, { "label": "completed_group_1", "mean": 0.17211595177650452, "median": 0.17136064171791077, "p05": 0.1543424427509308, "p95": 0.1928609162569046 }, { "label": "completed_group_2", "mean": 0.14605136215686798, "median": 0.14629822969436646, "p05": 0.13246256113052368, "p95": 0.15914371609687805 }, { "label": "completed_group_3", "mean": 0.16254930198192596, "median": 0.16272446513175964, "p05": 0.14075018465518951, "p95": 0.18463709950447083 }, { "label": "current_group_partial", "mean": 0.3255118429660797, "median": 0.3261105418205261, "p05": 0.3035835921764374, "p95": 0.3457673192024231 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.540909767150879, "group": 4, "latest_source_mass": 0.2995486855506897, "layer": 14, "max_source_mass": 0.2995486855506897, "mixer_index": 27, "normalized_entropy": 0.9574210693349581, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21921303868293762, "median": 0.21722173690795898, "p05": 0.1413341462612152, "p95": 0.2906968593597412 }, { "label": "completed_group_1", "mean": 0.10205383598804474, "median": 0.09984464198350906, "p05": 0.07455356419086456, "p95": 0.13362759351730347 }, { "label": "completed_group_2", "mean": 0.16381604969501495, "median": 0.16600549221038818, "p05": 0.11865930259227753, "p95": 0.2019941210746765 }, { "label": "completed_group_3", "mean": 0.215368390083313, "median": 0.2136734575033188, "p05": 0.1728287935256958, "p95": 0.2628125846385956 }, { "label": "current_group_partial", "mean": 0.2995486855506897, "median": 0.2982075810432434, "p05": 0.23863884806632996, "p95": 0.3662579655647278 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5142419338226318, "group": 4, "latest_source_mass": 0.369165301322937, "layer": 15, "max_source_mass": 0.369165301322937, "mixer_index": 28, "normalized_entropy": 0.940851412859105, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2000826597213745, "median": 0.20369741320610046, "p05": 0.15802279114723206, "p95": 0.24072687327861786 }, { "label": "completed_group_1", "mean": 0.15480534732341766, "median": 0.15228059887886047, "p05": 0.13543802499771118, "p95": 0.1815909892320633 }, { "label": "completed_group_2", "mean": 0.10827909409999847, "median": 0.10761409997940063, "p05": 0.09670238941907883, "p95": 0.12229038774967194 }, { "label": "completed_group_3", "mean": 0.16766759753227234, "median": 0.16637766361236572, "p05": 0.1398160457611084, "p95": 0.20042110979557037 }, { "label": "current_group_partial", "mean": 0.369165301322937, "median": 0.3697334825992584, "p05": 0.3279486298561096, "p95": 0.40613043308258057 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5333527326583862, "group": 4, "latest_source_mass": 0.29974091053009033, "layer": 15, "max_source_mass": 0.29974091053009033, "mixer_index": 29, "normalized_entropy": 0.9527256198031004, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21770668029785156, "median": 0.19699469208717346, "p05": 0.1311802864074707, "p95": 0.3720983564853668 }, { "label": "completed_group_1", "mean": 0.10525081306695938, "median": 0.10493603348731995, "p05": 0.08330865204334259, "p95": 0.12839283049106598 }, { "label": "completed_group_2", "mean": 0.15970060229301453, "median": 0.16236470639705658, "p05": 0.11309029906988144, "p95": 0.19861777126789093 }, { "label": "completed_group_3", "mean": 0.217601016163826, "median": 0.21904586255550385, "p05": 0.16268108785152435, "p95": 0.2699928283691406 }, { "label": "current_group_partial", "mean": 0.29974091053009033, "median": 0.30079299211502075, "p05": 0.22156940400600433, "p95": 0.37281978130340576 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.512608289718628, "group": 4, "latest_source_mass": 0.35851576924324036, "layer": 16, "max_source_mass": 0.35851576924324036, "mixer_index": 30, "normalized_entropy": 0.9398363727066501, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21392285823822021, "median": 0.22146853804588318, "p05": 0.1518818885087967, "p95": 0.25874194502830505 }, { "label": "completed_group_1", "mean": 0.1860550194978714, "median": 0.1838144212961197, "p05": 0.1660098135471344, "p95": 0.21311265230178833 }, { "label": "completed_group_2", "mean": 0.10291537642478943, "median": 0.1024642288684845, "p05": 0.09207107126712799, "p95": 0.11504144221544266 }, { "label": "completed_group_3", "mean": 0.1385909616947174, "median": 0.13646918535232544, "p05": 0.11220520734786987, "p95": 0.1728362739086151 }, { "label": "current_group_partial", "mean": 0.35851576924324036, "median": 0.3558155298233032, "p05": 0.31809645891189575, "p95": 0.40622422099113464 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5262645483016968, "group": 4, "latest_source_mass": 0.3142552077770233, "layer": 16, "max_source_mass": 0.3142552077770233, "mixer_index": 31, "normalized_entropy": 0.9483214832396903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20246261358261108, "median": 0.18459102511405945, "p05": 0.12276223301887512, "p95": 0.38892465829849243 }, { "label": "completed_group_1", "mean": 0.10530521720647812, "median": 0.10532709956169128, "p05": 0.0785030871629715, "p95": 0.1342671513557434 }, { "label": "completed_group_2", "mean": 0.16650916635990143, "median": 0.17111444473266602, "p05": 0.1085265576839447, "p95": 0.2094469517469406 }, { "label": "completed_group_3", "mean": 0.21146778762340546, "median": 0.2099919468164444, "p05": 0.15801574289798737, "p95": 0.26882117986679077 }, { "label": "current_group_partial", "mean": 0.3142552077770233, "median": 0.31609731912612915, "p05": 0.22627028822898865, "p95": 0.397154837846756 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5794764757156372, "group": 5, "latest_source_mass": 0.2760102152824402, "layer": 17, "max_source_mass": 0.2760102152824402, "mixer_index": 32, "normalized_entropy": 0.9813839126772218, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2210189402103424, "median": 0.22435861825942993, "p05": 0.17291823029518127, "p95": 0.262204647064209 }, { "label": "completed_group_1", "mean": 0.18839938938617706, "median": 0.18652752041816711, "p05": 0.16997642815113068, "p95": 0.2126670628786087 }, { "label": "completed_group_2", "mean": 0.14016608893871307, "median": 0.14067986607551575, "p05": 0.1265786588191986, "p95": 0.15091627836227417 }, { "label": "completed_group_3", "mean": 0.17440536618232727, "median": 0.17192018032073975, "p05": 0.15062645077705383, "p95": 0.20691406726837158 }, { "label": "completed_group_4", "mean": 0.2760102152824402, "median": 0.27511972188949585, "p05": 0.24412119388580322, "p95": 0.30901142954826355 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6602650880813599, "group": 5, "latest_source_mass": 0.3575916588306427, "layer": 17, "max_source_mass": 0.3575916588306427, "mixer_index": 33, "normalized_entropy": 0.9266115885502495, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12695768475532532, "median": 0.11564654856920242, "p05": 0.07159632444381714, "p95": 0.21949690580368042 }, { "label": "completed_group_1", "mean": 0.08036886155605316, "median": 0.07964898645877838, "p05": 0.06650256365537643, "p95": 0.09638062119483948 }, { "label": "completed_group_2", "mean": 0.12672865390777588, "median": 0.1287270039319992, "p05": 0.09939455986022949, "p95": 0.14712141454219818 }, { "label": "completed_group_3", "mean": 0.15391482412815094, "median": 0.15372109413146973, "p05": 0.13333335518836975, "p95": 0.17581847310066223 }, { "label": "completed_group_4", "mean": 0.154438316822052, "median": 0.15402798354625702, "p05": 0.12481103092432022, "p95": 0.18498417735099792 }, { "label": "current_group_partial", "mean": 0.3575916588306427, "median": 0.36001941561698914, "p05": 0.3110165596008301, "p95": 0.39699214696884155 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.5925545692443848, "group": 5, "latest_source_mass": 0.41908538341522217, "layer": 18, "max_source_mass": 0.41908538341522217, "mixer_index": 34, "normalized_entropy": 0.8888216284580353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14075040817260742, "median": 0.1455841362476349, "p05": 0.09744156897068024, "p95": 0.1818566620349884 }, { "label": "completed_group_1", "mean": 0.11028306186199188, "median": 0.10887216031551361, "p05": 0.09629634767770767, "p95": 0.12874113023281097 }, { "label": "completed_group_2", "mean": 0.07537610828876495, "median": 0.07517752051353455, "p05": 0.0666593387722969, "p95": 0.08491044491529465 }, { "label": "completed_group_3", "mean": 0.10244421660900116, "median": 0.10141433775424957, "p05": 0.08653581142425537, "p95": 0.12175601720809937 }, { "label": "completed_group_4", "mean": 0.1520608365535736, "median": 0.1513972282409668, "p05": 0.1301831305027008, "p95": 0.17589205503463745 }, { "label": "current_group_partial", "mean": 0.41908538341522217, "median": 0.4191858768463135, "p05": 0.3649080991744995, "p95": 0.4721684455871582 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5918848514556885, "group": 5, "latest_source_mass": 0.4151553511619568, "layer": 18, "max_source_mass": 0.4151553511619568, "mixer_index": 35, "normalized_entropy": 0.8884478518433735, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11773698031902313, "median": 0.10857899487018585, "p05": 0.06324320286512375, "p95": 0.22339528799057007 }, { "label": "completed_group_1", "mean": 0.07344591617584229, "median": 0.07228962332010269, "p05": 0.05982384458184242, "p95": 0.09060496836900711 }, { "label": "completed_group_2", "mean": 0.11976176500320435, "median": 0.12151201069355011, "p05": 0.08953659236431122, "p95": 0.14441059529781342 }, { "label": "completed_group_3", "mean": 0.14735516905784607, "median": 0.14620742201805115, "p05": 0.11767779290676117, "p95": 0.17902395129203796 }, { "label": "completed_group_4", "mean": 0.126544788479805, "median": 0.12564796209335327, "p05": 0.09858805686235428, "p95": 0.1567486822605133 }, { "label": "current_group_partial", "mean": 0.4151553511619568, "median": 0.4184337854385376, "p05": 0.3375611901283264, "p95": 0.4846380352973938 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.6208878755569458, "group": 5, "latest_source_mass": 0.3991435170173645, "layer": 19, "max_source_mass": 0.3991435170173645, "mixer_index": 36, "normalized_entropy": 0.9046347477964072, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14785674214363098, "median": 0.15170978009700775, "p05": 0.09146977216005325, "p95": 0.20172694325447083 }, { "label": "completed_group_1", "mean": 0.12492769956588745, "median": 0.12345598638057709, "p05": 0.10790181905031204, "p95": 0.14623555541038513 }, { "label": "completed_group_2", "mean": 0.08781818300485611, "median": 0.08839264512062073, "p05": 0.07453933358192444, "p95": 0.09894395619630814 }, { "label": "completed_group_3", "mean": 0.10019385814666748, "median": 0.09901996701955795, "p05": 0.08430047333240509, "p95": 0.12013883143663406 }, { "label": "completed_group_4", "mean": 0.14006000757217407, "median": 0.13918349146842957, "p05": 0.11816112697124481, "p95": 0.16475799679756165 }, { "label": "current_group_partial", "mean": 0.3991435170173645, "median": 0.39726996421813965, "p05": 0.34116071462631226, "p95": 0.4634247422218323 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5470075607299805, "group": 5, "latest_source_mass": 0.44330859184265137, "layer": 19, "max_source_mass": 0.44330859184265137, "mixer_index": 37, "normalized_entropy": 0.8634013589985261, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12043030560016632, "median": 0.1012125313282013, "p05": 0.061379045248031616, "p95": 0.25343188643455505 }, { "label": "completed_group_1", "mean": 0.06855648756027222, "median": 0.06761305779218674, "p05": 0.05337192490696907, "p95": 0.08790726959705353 }, { "label": "completed_group_2", "mean": 0.11753211170434952, "median": 0.1186976358294487, "p05": 0.08412563800811768, "p95": 0.14895018935203552 }, { "label": "completed_group_3", "mean": 0.1339646875858307, "median": 0.13253431022167206, "p05": 0.09857159107923508, "p95": 0.1730121374130249 }, { "label": "completed_group_4", "mean": 0.1162077933549881, "median": 0.11592038720846176, "p05": 0.08609852194786072, "p95": 0.14767511188983917 }, { "label": "current_group_partial", "mean": 0.44330859184265137, "median": 0.4434061646461487, "p05": 0.3389148414134979, "p95": 0.5490255355834961 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.683492660522461, "group": 5, "latest_source_mass": 0.3365415334701538, "layer": 20, "max_source_mass": 0.3365415334701538, "mixer_index": 38, "normalized_entropy": 0.9395751435586169, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1658952683210373, "median": 0.1726911962032318, "p05": 0.10183650255203247, "p95": 0.22522984445095062 }, { "label": "completed_group_1", "mean": 0.1435280442237854, "median": 0.14210543036460876, "p05": 0.12450659275054932, "p95": 0.16782894730567932 }, { "label": "completed_group_2", "mean": 0.09461991488933563, "median": 0.0948714017868042, "p05": 0.08168491721153259, "p95": 0.10723742842674255 }, { "label": "completed_group_3", "mean": 0.10784181207418442, "median": 0.10693192481994629, "p05": 0.08930590003728867, "p95": 0.12958656251430511 }, { "label": "completed_group_4", "mean": 0.15157344937324524, "median": 0.15053030848503113, "p05": 0.128616601228714, "p95": 0.17799487709999084 }, { "label": "current_group_partial", "mean": 0.3365415334701538, "median": 0.33349889516830444, "p05": 0.2778944969177246, "p95": 0.40724658966064453 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5540567636489868, "group": 5, "latest_source_mass": 0.42266085743904114, "layer": 20, "max_source_mass": 0.42266085743904114, "mixer_index": 39, "normalized_entropy": 0.8673355940563396, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12987135350704193, "median": 0.1083056628704071, "p05": 0.03783264383673668, "p95": 0.3301126956939697 }, { "label": "completed_group_1", "mean": 0.07178181409835815, "median": 0.07081792503595352, "p05": 0.05334944650530815, "p95": 0.09311570972204208 }, { "label": "completed_group_2", "mean": 0.12132491171360016, "median": 0.12282539904117584, "p05": 0.07926258444786072, "p95": 0.15763431787490845 }, { "label": "completed_group_3", "mean": 0.1441602110862732, "median": 0.14273977279663086, "p05": 0.09847753494977951, "p95": 0.19304539263248444 }, { "label": "completed_group_4", "mean": 0.11020088195800781, "median": 0.10934801399707794, "p05": 0.07139573246240616, "p95": 0.1517774760723114 }, { "label": "current_group_partial", "mean": 0.42266085743904114, "median": 0.42266106605529785, "p05": 0.28481048345565796, "p95": 0.5570361018180847 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7208337783813477, "group": 6, "latest_source_mass": 0.27070939540863037, "layer": 21, "max_source_mass": 0.27070939540863037, "mixer_index": 40, "normalized_entropy": 0.9604156182429642, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.18048501014709473, "median": 0.18919861316680908, "p05": 0.11199048161506653, "p95": 0.23807701468467712 }, { "label": "completed_group_1", "mean": 0.1685236394405365, "median": 0.1669977903366089, "p05": 0.14678362011909485, "p95": 0.19381441175937653 }, { "label": "completed_group_2", "mean": 0.08726631104946136, "median": 0.08729851245880127, "p05": 0.07629868388175964, "p95": 0.0980045348405838 }, { "label": "completed_group_3", "mean": 0.11160479485988617, "median": 0.1099197044968605, "p05": 0.09190923720598221, "p95": 0.13723208010196686 }, { "label": "completed_group_4", "mean": 0.18141084909439087, "median": 0.18020854890346527, "p05": 0.15656134486198425, "p95": 0.2108723670244217 }, { "label": "completed_group_5", "mean": 0.27070939540863037, "median": 0.2684924602508545, "p05": 0.2159801721572876, "p95": 0.33192309737205505 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6729483604431152, "group": 6, "latest_source_mass": 0.3803577423095703, "layer": 21, "max_source_mass": 0.3803577423095703, "mixer_index": 41, "normalized_entropy": 0.8597253893019091, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09229261428117752, "median": 0.07798201590776443, "p05": 0.04784023016691208, "p95": 0.20285779237747192 }, { "label": "completed_group_1", "mean": 0.051420148462057114, "median": 0.05041736364364624, "p05": 0.040625106543302536, "p95": 0.06547587364912033 }, { "label": "completed_group_2", "mean": 0.08079522103071213, "median": 0.08136115223169327, "p05": 0.05947277322411537, "p95": 0.10017222166061401 }, { "label": "completed_group_3", "mean": 0.09550285339355469, "median": 0.09445887804031372, "p05": 0.07334083318710327, "p95": 0.12144046276807785 }, { "label": "completed_group_4", "mean": 0.07590493559837341, "median": 0.07492390275001526, "p05": 0.055510398000478745, "p95": 0.10084333270788193 }, { "label": "completed_group_5", "mean": 0.22372648119926453, "median": 0.22069036960601807, "p05": 0.1407623589038849, "p95": 0.32290777564048767 }, { "label": "current_group_partial", "mean": 0.3803577423095703, "median": 0.37903881072998047, "p05": 0.2855539321899414, "p95": 0.476589173078537 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.710235595703125, "group": 6, "latest_source_mass": 0.4011716842651367, "layer": 22, "max_source_mass": 0.4011716842651367, "mixer_index": 42, "normalized_entropy": 0.8788872376935791, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11657392978668213, "median": 0.11877645552158356, "p05": 0.06819256395101547, "p95": 0.1684979349374771 }, { "label": "completed_group_1", "mean": 0.10617930442094803, "median": 0.10509790480136871, "p05": 0.090352863073349, "p95": 0.12587188184261322 }, { "label": "completed_group_2", "mean": 0.061112720519304276, "median": 0.061012350022792816, "p05": 0.05164862796664238, "p95": 0.07133162766695023 }, { "label": "completed_group_3", "mean": 0.06796726584434509, "median": 0.06712733209133148, "p05": 0.05444999784231186, "p95": 0.08481802046298981 }, { "label": "completed_group_4", "mean": 0.103337861597538, "median": 0.10192783921957016, "p05": 0.0832459107041359, "p95": 0.12805812060832977 }, { "label": "completed_group_5", "mean": 0.14365722239017487, "median": 0.14159655570983887, "p05": 0.10498540848493576, "p95": 0.19020715355873108 }, { "label": "current_group_partial", "mean": 0.4011716842651367, "median": 0.40084803104400635, "p05": 0.32083970308303833, "p95": 0.47994357347488403 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5640671253204346, "group": 6, "latest_source_mass": 0.4689176678657532, "layer": 22, "max_source_mass": 0.4689176678657532, "mixer_index": 43, "normalized_entropy": 0.8037715030571925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.090422622859478, "median": 0.0774901732802391, "p05": 0.04492149129509926, "p95": 0.1988787055015564 }, { "label": "completed_group_1", "mean": 0.04561382159590721, "median": 0.044634923338890076, "p05": 0.03501313924789429, "p95": 0.05900419130921364 }, { "label": "completed_group_2", "mean": 0.07565715163946152, "median": 0.07550699263811111, "p05": 0.053458306938409805, "p95": 0.09821777045726776 }, { "label": "completed_group_3", "mean": 0.08878682553768158, "median": 0.08692863583564758, "p05": 0.06425619125366211, "p95": 0.11807316541671753 }, { "label": "completed_group_4", "mean": 0.06555216014385223, "median": 0.06414324045181274, "p05": 0.04460844025015831, "p95": 0.09090004861354828 }, { "label": "completed_group_5", "mean": 0.165049746632576, "median": 0.15503892302513123, "p05": 0.08368627727031708, "p95": 0.2792147397994995 }, { "label": "current_group_partial", "mean": 0.4689176678657532, "median": 0.46879667043685913, "p05": 0.3151778280735016, "p95": 0.6170428991317749 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7556228637695312, "group": 6, "latest_source_mass": 0.3706088662147522, "layer": 23, "max_source_mass": 0.3706088662147522, "mixer_index": 44, "normalized_entropy": 0.9022116795175967, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13281464576721191, "median": 0.13658174872398376, "p05": 0.08060654997825623, "p95": 0.17402096092700958 }, { "label": "completed_group_1", "mean": 0.12175846099853516, "median": 0.12066285312175751, "p05": 0.10334118455648422, "p95": 0.14384035766124725 }, { "label": "completed_group_2", "mean": 0.07235272228717804, "median": 0.07218228280544281, "p05": 0.059715576469898224, "p95": 0.08584222197532654 }, { "label": "completed_group_3", "mean": 0.0707377940416336, "median": 0.06942892074584961, "p05": 0.05500924959778786, "p95": 0.09052760899066925 }, { "label": "completed_group_4", "mean": 0.10551950335502625, "median": 0.103996641933918, "p05": 0.08408617228269577, "p95": 0.13205930590629578 }, { "label": "completed_group_5", "mean": 0.12620802223682404, "median": 0.12379512935876846, "p05": 0.08937962353229523, "p95": 0.17139491438865662 }, { "label": "current_group_partial", "mean": 0.3706088662147522, "median": 0.3705657124519348, "p05": 0.282781720161438, "p95": 0.4561466872692108 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.594892978668213, "group": 6, "latest_source_mass": 0.4636557698249817, "layer": 23, "max_source_mass": 0.4636557698249817, "mixer_index": 45, "normalized_entropy": 0.8196128579947488, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10721131414175034, "median": 0.10240983963012695, "p05": 0.05563424155116081, "p95": 0.18401460349559784 }, { "label": "completed_group_1", "mean": 0.053925663232803345, "median": 0.0527028888463974, "p05": 0.04038037359714508, "p95": 0.07054222375154495 }, { "label": "completed_group_2", "mean": 0.07673212885856628, "median": 0.07705660909414291, "p05": 0.05211106687784195, "p95": 0.10031480342149734 }, { "label": "completed_group_3", "mean": 0.08865120261907578, "median": 0.08699032664299011, "p05": 0.06242028623819351, "p95": 0.11899784207344055 }, { "label": "completed_group_4", "mean": 0.06888366490602493, "median": 0.06751501560211182, "p05": 0.04784276708960533, "p95": 0.09609999507665634 }, { "label": "completed_group_5", "mean": 0.14094024896621704, "median": 0.13063763082027435, "p05": 0.07297889143228531, "p95": 0.24012833833694458 }, { "label": "current_group_partial", "mean": 0.4636557698249817, "median": 0.46312135457992554, "p05": 0.31259939074516296, "p95": 0.6109440326690674 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7775511741638184, "group": 6, "latest_source_mass": 0.3569435477256775, "layer": 24, "max_source_mass": 0.3569435477256775, "mixer_index": 46, "normalized_entropy": 0.9134806018801903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13637229800224304, "median": 0.13456633687019348, "p05": 0.09586701542139053, "p95": 0.18876218795776367 }, { "label": "completed_group_1", "mean": 0.10478455573320389, "median": 0.10326151549816132, "p05": 0.0880400538444519, "p95": 0.12566551566123962 }, { "label": "completed_group_2", "mean": 0.07710614800453186, "median": 0.07678486406803131, "p05": 0.06292064487934113, "p95": 0.09218747913837433 }, { "label": "completed_group_3", "mean": 0.08548048138618469, "median": 0.08430111408233643, "p05": 0.0665682703256607, "p95": 0.10857507586479187 }, { "label": "completed_group_4", "mean": 0.11000140011310577, "median": 0.10904969274997711, "p05": 0.0881001353263855, "p95": 0.13561299443244934 }, { "label": "completed_group_5", "mean": 0.12931162118911743, "median": 0.12692990899085999, "p05": 0.08648453652858734, "p95": 0.1814313679933548 }, { "label": "current_group_partial", "mean": 0.3569435477256775, "median": 0.3568228483200073, "p05": 0.26841166615486145, "p95": 0.4493965804576874 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.654036045074463, "group": 6, "latest_source_mass": 0.4265251159667969, "layer": 24, "max_source_mass": 0.4265251159667969, "mixer_index": 47, "normalized_entropy": 0.8500063817835848, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10232305526733398, "median": 0.08623291552066803, "p05": 0.04679499566555023, "p95": 0.23077142238616943 }, { "label": "completed_group_1", "mean": 0.061968062072992325, "median": 0.060845747590065, "p05": 0.0479469932615757, "p95": 0.07941868156194687 }, { "label": "completed_group_2", "mean": 0.08900615572929382, "median": 0.08864577114582062, "p05": 0.0641680359840393, "p95": 0.11413820832967758 }, { "label": "completed_group_3", "mean": 0.1005433052778244, "median": 0.09813467413187027, "p05": 0.0731433779001236, "p95": 0.1343638002872467 }, { "label": "completed_group_4", "mean": 0.07864196598529816, "median": 0.07639941573143005, "p05": 0.05487418174743652, "p95": 0.10941869765520096 }, { "label": "completed_group_5", "mean": 0.14099235832691193, "median": 0.12885531783103943, "p05": 0.07315157353878021, "p95": 0.24481485784053802 }, { "label": "current_group_partial", "mean": 0.4265251159667969, "median": 0.42903727293014526, "p05": 0.2803359627723694, "p95": 0.5635722875595093 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.84956693649292, "group": 7, "latest_source_mass": 0.28775152564048767, "layer": 25, "max_source_mass": 0.28775152564048767, "mixer_index": 48, "normalized_entropy": 0.9504893827656096, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14851760864257812, "median": 0.14813844859600067, "p05": 0.10312555730342865, "p95": 0.20065288245677948 }, { "label": "completed_group_1", "mean": 0.11630287766456604, "median": 0.11500488221645355, "p05": 0.09825178980827332, "p95": 0.13750022649765015 }, { "label": "completed_group_2", "mean": 0.08512166142463684, "median": 0.08458003401756287, "p05": 0.0710662305355072, "p95": 0.10092862695455551 }, { "label": "completed_group_3", "mean": 0.09215595573186874, "median": 0.09079311043024063, "p05": 0.07304700464010239, "p95": 0.11515989899635315 }, { "label": "completed_group_4", "mean": 0.1263950914144516, "median": 0.12563186883926392, "p05": 0.10441911220550537, "p95": 0.1519687920808792 }, { "label": "completed_group_5", "mean": 0.14375528693199158, "median": 0.1422564834356308, "p05": 0.10313796997070312, "p95": 0.1929876059293747 }, { "label": "completed_group_6", "mean": 0.28775152564048767, "median": 0.28676605224609375, "p05": 0.21690022945404053, "p95": 0.3624127507209778 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.7161309719085693, "group": 7, "latest_source_mass": 0.32536065578460693, "layer": 25, "max_source_mass": 0.32536065578460693, "mixer_index": 49, "normalized_entropy": 0.8252845475628168, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0837959572672844, "median": 0.0849243700504303, "p05": 0.0271806251257658, "p95": 0.15180703997612 }, { "label": "completed_group_1", "mean": 0.04434927552938461, "median": 0.04302988201379776, "p05": 0.03217232972383499, "p95": 0.06002792343497276 }, { "label": "completed_group_2", "mean": 0.056708935648202896, "median": 0.05563671886920929, "p05": 0.03992946073412895, "p95": 0.07731831818819046 }, { "label": "completed_group_3", "mean": 0.06495873630046844, "median": 0.06223570555448532, "p05": 0.04220382124185562, "p95": 0.0949486792087555 }, { "label": "completed_group_4", "mean": 0.04885463789105415, "median": 0.046898528933525085, "p05": 0.030923232436180115, "p95": 0.0743112787604332 }, { "label": "completed_group_5", "mean": 0.09997115284204483, "median": 0.08650931715965271, "p05": 0.04111029952764511, "p95": 0.20292627811431885 }, { "label": "completed_group_6", "mean": 0.2760006785392761, "median": 0.27142390608787537, "p05": 0.13761089742183685, "p95": 0.42605364322662354 }, { "label": "current_group_partial", "mean": 0.32536065578460693, "median": 0.3348004221916199, "p05": 0.10654139518737793, "p95": 0.5236068964004517 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9285151958465576, "group": 7, "latest_source_mass": 0.2804902195930481, "layer": 26, "max_source_mass": 0.2804902195930481, "mixer_index": 50, "normalized_entropy": 0.9274197697756124, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10564712435007095, "median": 0.10497278720140457, "p05": 0.04608207941055298, "p95": 0.16350959241390228 }, { "label": "completed_group_1", "mean": 0.09173665940761566, "median": 0.09041133522987366, "p05": 0.07509034872055054, "p95": 0.11238400638103485 }, { "label": "completed_group_2", "mean": 0.07135222852230072, "median": 0.06981296837329865, "p05": 0.0522611029446125, "p95": 0.09555281698703766 }, { "label": "completed_group_3", "mean": 0.07045470923185349, "median": 0.06927657127380371, "p05": 0.055566973984241486, "p95": 0.08908769488334656 }, { "label": "completed_group_4", "mean": 0.09785081446170807, "median": 0.0971219539642334, "p05": 0.07590530812740326, "p95": 0.12285276502370834 }, { "label": "completed_group_5", "mean": 0.10425601899623871, "median": 0.10151226818561554, "p05": 0.06786561012268066, "p95": 0.15156632661819458 }, { "label": "completed_group_6", "mean": 0.1782122254371643, "median": 0.1762663722038269, "p05": 0.11771149188280106, "p95": 0.24932436645030975 }, { "label": "current_group_partial", "mean": 0.2804902195930481, "median": 0.2783043086528778, "p05": 0.14811429381370544, "p95": 0.4168073534965515 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7219383716583252, "group": 7, "latest_source_mass": 0.3661728501319885, "layer": 26, "max_source_mass": 0.3661728501319885, "mixer_index": 51, "normalized_entropy": 0.8280773165026276, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0923302173614502, "median": 0.08522969484329224, "p05": 0.056062981486320496, "p95": 0.1509525179862976 }, { "label": "completed_group_1", "mean": 0.04646994173526764, "median": 0.04543843865394592, "p05": 0.032575178891420364, "p95": 0.06306323409080505 }, { "label": "completed_group_2", "mean": 0.05750556290149689, "median": 0.05649096518754959, "p05": 0.03895717114210129, "p95": 0.07940060645341873 }, { "label": "completed_group_3", "mean": 0.06549686938524246, "median": 0.062135741114616394, "p05": 0.04169235751032829, "p95": 0.09948549419641495 }, { "label": "completed_group_4", "mean": 0.054404959082603455, "median": 0.05241036415100098, "p05": 0.03378590941429138, "p95": 0.08157843351364136 }, { "label": "completed_group_5", "mean": 0.08427833020687103, "median": 0.07576382160186768, "p05": 0.041103728115558624, "p95": 0.15204203128814697 }, { "label": "completed_group_6", "mean": 0.2333412617444992, "median": 0.22215056419372559, "p05": 0.10060560703277588, "p95": 0.3983197808265686 }, { "label": "current_group_partial", "mean": 0.3661728501319885, "median": 0.3673480749130249, "p05": 0.16410952806472778, "p95": 0.5635213255882263 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9470824003219604, "group": 7, "latest_source_mass": 0.26050227880477905, "layer": 27, "max_source_mass": 0.26050227880477905, "mixer_index": 52, "normalized_entropy": 0.9363487077155573, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10804469138383865, "median": 0.10379163920879364, "p05": 0.05421917140483856, "p95": 0.17028658092021942 }, { "label": "completed_group_1", "mean": 0.07850409299135208, "median": 0.0776081532239914, "p05": 0.063654825091362, "p95": 0.09574449062347412 }, { "label": "completed_group_2", "mean": 0.07298290729522705, "median": 0.07200999557971954, "p05": 0.05533483251929283, "p95": 0.09299935400485992 }, { "label": "completed_group_3", "mean": 0.08511961251497269, "median": 0.08372659981250763, "p05": 0.06449062377214432, "p95": 0.11086944490671158 }, { "label": "completed_group_4", "mean": 0.09968686103820801, "median": 0.09877584874629974, "p05": 0.07606595754623413, "p95": 0.1266063004732132 }, { "label": "completed_group_5", "mean": 0.11502514779567719, "median": 0.11171717941761017, "p05": 0.07302433997392654, "p95": 0.1681303083896637 }, { "label": "completed_group_6", "mean": 0.18013443052768707, "median": 0.1765173077583313, "p05": 0.11514104157686234, "p95": 0.2599799633026123 }, { "label": "current_group_partial", "mean": 0.26050227880477905, "median": 0.2575443983078003, "p05": 0.13687226176261902, "p95": 0.39756521582603455 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7855088710784912, "group": 7, "latest_source_mass": 0.28804832696914673, "layer": 27, "max_source_mass": 0.28804832696914673, "mixer_index": 53, "normalized_entropy": 0.858648264589397, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08493785560131073, "median": 0.05951457470655441, "p05": 0.02441210113465786, "p95": 0.22703105211257935 }, { "label": "completed_group_1", "mean": 0.05972691997885704, "median": 0.058127306401729584, "p05": 0.03591413050889969, "p95": 0.08903861790895462 }, { "label": "completed_group_2", "mean": 0.07017223536968231, "median": 0.06906847655773163, "p05": 0.04151676222681999, "p95": 0.10273467004299164 }, { "label": "completed_group_3", "mean": 0.07963316142559052, "median": 0.07547932863235474, "p05": 0.042773112654685974, "p95": 0.12874338030815125 }, { "label": "completed_group_4", "mean": 0.0645163506269455, "median": 0.061876002699136734, "p05": 0.0366181954741478, "p95": 0.10098200291395187 }, { "label": "completed_group_5", "mean": 0.11586236953735352, "median": 0.10182450711727142, "p05": 0.048887670040130615, "p95": 0.2313569337129593 }, { "label": "completed_group_6", "mean": 0.23710274696350098, "median": 0.22610372304916382, "p05": 0.10011275857686996, "p95": 0.4084784686565399 }, { "label": "current_group_partial", "mean": 0.28804832696914673, "median": 0.27438053488731384, "p05": 0.07688213139772415, "p95": 0.5529112815856934 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.973182201385498, "group": 7, "latest_source_mass": 0.23531918227672577, "layer": 28, "max_source_mass": 0.23531918227672577, "mixer_index": 54, "normalized_entropy": 0.9489000589030754, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.104981929063797, "median": 0.09386023879051208, "p05": 0.04212673753499985, "p95": 0.18614700436592102 }, { "label": "completed_group_1", "mean": 0.0840599462389946, "median": 0.08250434696674347, "p05": 0.05170809105038643, "p95": 0.12236068397760391 }, { "label": "completed_group_2", "mean": 0.09310521930456161, "median": 0.0931512713432312, "p05": 0.06374019384384155, "p95": 0.12329407781362534 }, { "label": "completed_group_3", "mean": 0.11566174775362015, "median": 0.11439734697341919, "p05": 0.07886623591184616, "p95": 0.1571493148803711 }, { "label": "completed_group_4", "mean": 0.0963074117898941, "median": 0.09477876126766205, "p05": 0.06686149537563324, "p95": 0.13022682070732117 }, { "label": "completed_group_5", "mean": 0.1096896305680275, "median": 0.10702179372310638, "p05": 0.06579374521970749, "p95": 0.1643676459789276 }, { "label": "completed_group_6", "mean": 0.16087493300437927, "median": 0.15455034375190735, "p05": 0.09876372665166855, "p95": 0.24324838817119598 }, { "label": "current_group_partial", "mean": 0.23531918227672577, "median": 0.22382870316505432, "p05": 0.11656329780817032, "p95": 0.38286590576171875 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.802741289138794, "group": 7, "latest_source_mass": 0.2497689425945282, "layer": 28, "max_source_mass": 0.2913346588611603, "mixer_index": 55, "normalized_entropy": 0.8669353059487718, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08190082013607025, "median": 0.06415966153144836, "p05": 0.039168842136859894, "p95": 0.23471535742282867 }, { "label": "completed_group_1", "mean": 0.05632811039686203, "median": 0.055233441293239594, "p05": 0.03936203569173813, "p95": 0.076178178191185 }, { "label": "completed_group_2", "mean": 0.07102417945861816, "median": 0.06932097673416138, "p05": 0.04159427434206009, "p95": 0.1046731173992157 }, { "label": "completed_group_3", "mean": 0.08421063423156738, "median": 0.0812690332531929, "p05": 0.0486646331846714, "p95": 0.12889066338539124 }, { "label": "completed_group_4", "mean": 0.06341387331485748, "median": 0.061601825058460236, "p05": 0.03807957097887993, "p95": 0.09577237069606781 }, { "label": "completed_group_5", "mean": 0.10201875865459442, "median": 0.09330011904239655, "p05": 0.05116701498627663, "p95": 0.18108239769935608 }, { "label": "completed_group_6", "mean": 0.2913346588611603, "median": 0.28823426365852356, "p05": 0.13295966386795044, "p95": 0.4586564898490906 }, { "label": "current_group_partial", "mean": 0.2497689425945282, "median": 0.234481081366539, "p05": 0.09318265318870544, "p95": 0.4530518054962158 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.945815086364746, "group": 8, "latest_source_mass": 0.153213769197464, "layer": 29, "max_source_mass": 0.2451319396495819, "mixer_index": 56, "normalized_entropy": 0.9357392585284499, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1064736545085907, "median": 0.0881282389163971, "p05": 0.029344500973820686, "p95": 0.23970629274845123 }, { "label": "completed_group_1", "mean": 0.07925166189670563, "median": 0.07691921293735504, "p05": 0.062442928552627563, "p95": 0.1030145138502121 }, { "label": "completed_group_2", "mean": 0.07823626697063446, "median": 0.07715678215026855, "p05": 0.059018202126026154, "p95": 0.10055352747440338 }, { "label": "completed_group_3", "mean": 0.09447507560253143, "median": 0.09318602085113525, "p05": 0.0684189647436142, "p95": 0.12381350249052048 }, { "label": "completed_group_4", "mean": 0.09724453091621399, "median": 0.09628183394670486, "p05": 0.0699525699019432, "p95": 0.12776269018650055 }, { "label": "completed_group_5", "mean": 0.1459731012582779, "median": 0.14035329222679138, "p05": 0.08667999505996704, "p95": 0.2258891463279724 }, { "label": "completed_group_6", "mean": 0.2451319396495819, "median": 0.24505369365215302, "p05": 0.16207969188690186, "p95": 0.3307104706764221 }, { "label": "completed_group_7", "mean": 0.153213769197464, "median": 0.13417893648147583, "p05": 0.05588865652680397, "p95": 0.30438560247421265 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8717057704925537, "group": 8, "latest_source_mass": 0.3048541843891144, "layer": 29, "max_source_mass": 0.3048541843891144, "mixer_index": 57, "normalized_entropy": 0.8518500065030654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07336360216140747, "median": 0.0611552819609642, "p05": 0.024239415302872658, "p95": 0.16916418075561523 }, { "label": "completed_group_1", "mean": 0.05720400810241699, "median": 0.05402737483382225, "p05": 0.0314149409532547, "p95": 0.09488057345151901 }, { "label": "completed_group_2", "mean": 0.057025715708732605, "median": 0.05547948181629181, "p05": 0.03480810672044754, "p95": 0.08415934443473816 }, { "label": "completed_group_3", "mean": 0.06655597686767578, "median": 0.06265220046043396, "p05": 0.035200610756874084, "p95": 0.10949824750423431 }, { "label": "completed_group_4", "mean": 0.05032068490982056, "median": 0.047904569655656815, "p05": 0.02614513412117958, "p95": 0.08244554698467255 }, { "label": "completed_group_5", "mean": 0.08329877257347107, "median": 0.0731179416179657, "p05": 0.02939312346279621, "p95": 0.17248544096946716 }, { "label": "completed_group_6", "mean": 0.1901058554649353, "median": 0.18234091997146606, "p05": 0.08719424903392792, "p95": 0.317789226770401 }, { "label": "completed_group_7", "mean": 0.11727118492126465, "median": 0.09272899478673935, "p05": 0.031939513981342316, "p95": 0.2891285717487335 }, { "label": "current_group_partial", "mean": 0.3048541843891144, "median": 0.29432767629623413, "p05": 0.07913826406002045, "p95": 0.5632759928703308 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.09096622467041, "group": 8, "latest_source_mass": 0.13795366883277893, "layer": 30, "max_source_mass": 0.17320571839809418, "mixer_index": 58, "normalized_entropy": 0.951639739623416, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10326676070690155, "median": 0.09009090811014175, "p05": 0.01757553033530712, "p95": 0.2674753665924072 }, { "label": "completed_group_1", "mean": 0.08232800662517548, "median": 0.08093428611755371, "p05": 0.06430500745773315, "p95": 0.10457758605480194 }, { "label": "completed_group_2", "mean": 0.07164021581411362, "median": 0.0711187869310379, "p05": 0.05190654471516609, "p95": 0.09468771517276764 }, { "label": "completed_group_3", "mean": 0.08822821825742722, "median": 0.08667590469121933, "p05": 0.06148773431777954, "p95": 0.12025904655456543 }, { "label": "completed_group_4", "mean": 0.10803373157978058, "median": 0.10627314448356628, "p05": 0.07042569667100906, "p95": 0.1514367014169693 }, { "label": "completed_group_5", "mean": 0.13578416407108307, "median": 0.12929600477218628, "p05": 0.08227657526731491, "p95": 0.21117588877677917 }, { "label": "completed_group_6", "mean": 0.17320571839809418, "median": 0.1694163680076599, "p05": 0.11322733014822006, "p95": 0.24482715129852295 }, { "label": "completed_group_7", "mean": 0.09955951571464539, "median": 0.08973086625337601, "p05": 0.037010837346315384, "p95": 0.1912725567817688 }, { "label": "current_group_partial", "mean": 0.13795366883277893, "median": 0.1256815791130066, "p05": 0.053015124052762985, "p95": 0.2601332366466522 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.96881103515625, "group": 8, "latest_source_mass": 0.21570813655853271, "layer": 30, "max_source_mass": 0.21570813655853271, "mixer_index": 59, "normalized_entropy": 0.896044517007504, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08843819797039032, "median": 0.08496083319187164, "p05": 0.03690534085035324, "p95": 0.1522325873374939 }, { "label": "completed_group_1", "mean": 0.06052382290363312, "median": 0.058369796723127365, "p05": 0.03549504280090332, "p95": 0.09154202789068222 }, { "label": "completed_group_2", "mean": 0.0642135739326477, "median": 0.06355445086956024, "p05": 0.03948400169610977, "p95": 0.09175601601600647 }, { "label": "completed_group_3", "mean": 0.07744920998811722, "median": 0.07566337287425995, "p05": 0.04564609378576279, "p95": 0.11733324825763702 }, { "label": "completed_group_4", "mean": 0.051616813987493515, "median": 0.049985647201538086, "p05": 0.03238597884774208, "p95": 0.07699409872293472 }, { "label": "completed_group_5", "mean": 0.06631231307983398, "median": 0.061356961727142334, "p05": 0.030705859884619713, "p95": 0.1188126802444458 }, { "label": "completed_group_6", "mean": 0.16417551040649414, "median": 0.15454982221126556, "p05": 0.07227650284767151, "p95": 0.28998225927352905 }, { "label": "completed_group_7", "mean": 0.2115624099969864, "median": 0.19997704029083252, "p05": 0.09131578356027603, "p95": 0.3709705173969269 }, { "label": "current_group_partial", "mean": 0.21570813655853271, "median": 0.2005728930234909, "p05": 0.06839637458324432, "p95": 0.4193759858608246 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.124063014984131, "group": 8, "latest_source_mass": 0.1258423924446106, "layer": 31, "max_source_mass": 0.17017140984535217, "mixer_index": 60, "normalized_entropy": 0.9667027380329118, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09220977127552032, "median": 0.09152225404977798, "p05": 0.03251190483570099, "p95": 0.17071731388568878 }, { "label": "completed_group_1", "mean": 0.06839152425527573, "median": 0.06702418625354767, "p05": 0.04446800425648689, "p95": 0.0977444052696228 }, { "label": "completed_group_2", "mean": 0.10039852559566498, "median": 0.09736946225166321, "p05": 0.06476961821317673, "p95": 0.14545989036560059 }, { "label": "completed_group_3", "mean": 0.10902463644742966, "median": 0.10749617218971252, "p05": 0.07769118249416351, "p95": 0.14509552717208862 }, { "label": "completed_group_4", "mean": 0.0849665105342865, "median": 0.08355648815631866, "p05": 0.059977494180202484, "p95": 0.11480176448822021 }, { "label": "completed_group_5", "mean": 0.12137698382139206, "median": 0.11852307617664337, "p05": 0.07902064919471741, "p95": 0.17207489907741547 }, { "label": "completed_group_6", "mean": 0.17017140984535217, "median": 0.16817378997802734, "p05": 0.11521546542644501, "p95": 0.23236754536628723 }, { "label": "completed_group_7", "mean": 0.12761825323104858, "median": 0.1237221211194992, "p05": 0.062459878623485565, "p95": 0.2092883139848709 }, { "label": "current_group_partial", "mean": 0.1258423924446106, "median": 0.11723431199789047, "p05": 0.06504593789577484, "p95": 0.21421310305595398 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9633058309555054, "group": 8, "latest_source_mass": 0.15763342380523682, "layer": 31, "max_source_mass": 0.23511825501918793, "mixer_index": 61, "normalized_entropy": 0.8935389906004497, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.0846441388130188, "median": 0.06999644637107849, "p05": 0.04030589759349823, "p95": 0.20743247866630554 }, { "label": "completed_group_1", "mean": 0.05872901901602745, "median": 0.05653517693281174, "p05": 0.03710630163550377, "p95": 0.08772030472755432 }, { "label": "completed_group_2", "mean": 0.07210008800029755, "median": 0.07183568179607391, "p05": 0.03840115666389465, "p95": 0.10702595114707947 }, { "label": "completed_group_3", "mean": 0.07792744785547256, "median": 0.0762966051697731, "p05": 0.04050692170858383, "p95": 0.122304767370224 }, { "label": "completed_group_4", "mean": 0.059186771512031555, "median": 0.057271093130111694, "p05": 0.03537197411060333, "p95": 0.08958204090595245 }, { "label": "completed_group_5", "mean": 0.07445863634347916, "median": 0.06928195059299469, "p05": 0.03154807910323143, "p95": 0.1351260244846344 }, { "label": "completed_group_6", "mean": 0.23511825501918793, "median": 0.22703276574611664, "p05": 0.0953572615981102, "p95": 0.3982555568218231 }, { "label": "completed_group_7", "mean": 0.18020221590995789, "median": 0.16846294701099396, "p05": 0.07464539259672165, "p95": 0.324230432510376 }, { "label": "current_group_partial", "mean": 0.15763342380523682, "median": 0.12027859687805176, "p05": 0.036883872002363205, "p95": 0.4093724191188812 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1238560676574707, "group": 8, "latest_source_mass": 0.11516246944665909, "layer": 32, "max_source_mass": 0.1766587197780609, "mixer_index": 62, "normalized_entropy": 0.966608552245626, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10231245309114456, "median": 0.10458895564079285, "p05": 0.025718798860907555, "p95": 0.1875312179327011 }, { "label": "completed_group_1", "mean": 0.07575537264347076, "median": 0.07447351515293121, "p05": 0.05878028646111488, "p95": 0.09651422500610352 }, { "label": "completed_group_2", "mean": 0.08897508680820465, "median": 0.08751188218593597, "p05": 0.06392349302768707, "p95": 0.11800213158130646 }, { "label": "completed_group_3", "mean": 0.0967075377702713, "median": 0.09579810500144958, "p05": 0.06915527582168579, "p95": 0.12801720201969147 }, { "label": "completed_group_4", "mean": 0.09801244735717773, "median": 0.09654392302036285, "p05": 0.07431379705667496, "p95": 0.12754815816879272 }, { "label": "completed_group_5", "mean": 0.12438583374023438, "median": 0.12010829150676727, "p05": 0.08193086087703705, "p95": 0.1824759691953659 }, { "label": "completed_group_6", "mean": 0.1766587197780609, "median": 0.17520052194595337, "p05": 0.12216726690530777, "p95": 0.2341049611568451 }, { "label": "completed_group_7", "mean": 0.12203006446361542, "median": 0.1200856938958168, "p05": 0.05817953124642372, "p95": 0.19647708535194397 }, { "label": "current_group_partial", "mean": 0.11516246944665909, "median": 0.10430245846509933, "p05": 0.05224763602018356, "p95": 0.2294922173023224 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0835556983947754, "group": 8, "latest_source_mass": 0.09660109132528305, "layer": 32, "max_source_mass": 0.22511571645736694, "mixer_index": 63, "normalized_entropy": 0.9482670637704002, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09133295714855194, "median": 0.08019149303436279, "p05": 0.04441600665450096, "p95": 0.19931957125663757 }, { "label": "completed_group_1", "mean": 0.07054752856492996, "median": 0.06906121969223022, "p05": 0.05099350959062576, "p95": 0.09504306316375732 }, { "label": "completed_group_2", "mean": 0.09000501036643982, "median": 0.0901019275188446, "p05": 0.05998067185282707, "p95": 0.11849517375230789 }, { "label": "completed_group_3", "mean": 0.10956105589866638, "median": 0.1064089685678482, "p05": 0.0759081020951271, "p95": 0.1509535312652588 }, { "label": "completed_group_4", "mean": 0.07114207744598389, "median": 0.06995616853237152, "p05": 0.04991999268531799, "p95": 0.09545011073350906 }, { "label": "completed_group_5", "mean": 0.11565887182950974, "median": 0.11069735884666443, "p05": 0.06978912651538849, "p95": 0.17647549510002136 }, { "label": "completed_group_6", "mean": 0.22511571645736694, "median": 0.22372552752494812, "p05": 0.11895382404327393, "p95": 0.33412501215934753 }, { "label": "completed_group_7", "mean": 0.13003569841384888, "median": 0.12226496636867523, "p05": 0.06884703785181046, "p95": 0.21555939316749573 }, { "label": "current_group_partial", "mean": 0.09660109132528305, "median": 0.09000182151794434, "p05": 0.051288168877363205, "p95": 0.16580010950565338 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.679484486579895, "group": null, "latest_source_mass": 0.2827349305152893, "layer": null, "max_source_mass": 0.32717907428741455, "mixer_index": null, "normalized_entropy": 0.7643663300981273, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.08993907272815704, "median": 0.09290459752082825, "p05": 0.022419869899749756, "p95": 0.15114322304725647 }, { "label": "completed_group_1", "mean": 0.043493133038282394, "median": 0.042318835854530334, "p05": 0.026918282732367516, "p95": 0.06356139481067657 }, { "label": "completed_group_2", "mean": 0.04922422394156456, "median": 0.047891765832901, "p05": 0.03095712885260582, "p95": 0.07266643643379211 }, { "label": "completed_group_3", "mean": 0.037924449890851974, "median": 0.03562554344534874, "p05": 0.020909149199724197, "p95": 0.06224166974425316 }, { "label": "completed_group_4", "mean": 0.03939433768391609, "median": 0.036868683993816376, "p05": 0.020651251077651978, "p95": 0.06597590446472168 }, { "label": "completed_group_5", "mean": 0.03898489475250244, "median": 0.03362073004245758, "p05": 0.014276490546762943, "p95": 0.08282073587179184 }, { "label": "completed_group_6", "mean": 0.09112587571144104, "median": 0.07136045396327972, "p05": 0.02230823040008545, "p95": 0.22286513447761536 }, { "label": "completed_group_7", "mean": 0.32717907428741455, "median": 0.3213679790496826, "p05": 0.08271751552820206, "p95": 0.6128557920455933 }, { "label": "completed_group_8", "mean": 0.2827349305152893, "median": 0.2627606987953186, "p05": 0.05262754485011101, "p95": 0.571351170539856 } ], "sources": 9 } ], "mode": "uniform_value_backward", "optimizer_state_after": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "optimizer_state_before": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08626964688301086, 0.16337208449840546, 0.15584443509578705, 0.04305505007505417, 0.1360139101743698, 0.03393339365720749, 0.03730965405702591, 0.037494905292987823, 0.05234000086784363, 0.04223497211933136, 0.041619230061769485, 0.05722156912088394, 0.062187470495700836, 0.0369485504925251, 0.04293901473283768, 0.045369699597358704, 0.04149825870990753, 0.02440357394516468, 0.026009807363152504, 0.03634416684508324, 0.038355328142642975, 0.02631118707358837, 0.033921729773283005, 0.02731483057141304, 0.027557097375392914, 0.024315863847732544, 0.02051021158695221, 0.018631720915436745, 0.019497457891702652, 0.021715763956308365, 0.017114246264100075, 0.020576251670718193 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.943075613387919e-06, "first_to_last_ratio": 2.307872264355047, "imbalance_abs_log_ratio": 0.8363260023807616, "last_quartile_mean": 3.875030586186767e-06, "mean": 6.099237623402587e-06, "normalized": [ 1.6993132354180294, 1.6239172875947692, 1.5554831244671488, 1.442125192626584, 1.4104481551972334, 1.403523053069228, 1.3221099712249675, 1.2731697595883604, 1.2000943569582707, 1.1235452195545337, 1.0767048612407788, 1.0261714183282156, 1.0093778108098996, 0.9743388011452827, 0.9573184625534406, 0.928757504863306, 0.9116899710167641, 0.9026661339368709, 0.886533931006931, 0.8702403098677887, 0.8533876523625498, 0.8403774938169696, 0.8216913043945464, 0.8043724353132603, 0.7863149194889869, 0.7481207616594623, 0.7006856536668127, 0.6618189820973319, 0.6190766705086127, 0.569713863763572, 0.5180828888655294, 0.4788288135939631 ], "peak_layer": 1, "peak_normalized": 1.6993132354180294, "population_cv": 0.3198013047414009, "rest_mean": 6.301157672741856e-06, "spike_contrast": 0.7949125568843288, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.008869356970535e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.0364515219407622e-05, 9.904657417791896e-06, 9.487261195317842e-06, 8.795864232524764e-06, 8.602658454037737e-06, 8.5604206105927e-06, 8.063862878771033e-06, 7.765364898659755e-06, 7.319660653593019e-06, 6.8527692747011315e-06, 6.567078798980219e-06, 6.258863322727848e-06, 6.156435119919479e-06, 5.94272387388628e-06, 5.838912784383865e-06, 5.664712716679787e-06, 5.5606137721042614e-06, 5.505575245479122e-06, 5.407181106420467e-06, 5.307802439347142e-06, 5.205014076636871e-06, 5.125662028149236e-06, 5.011690518585965e-06, 4.906058620690601e-06, 4.795921540790005e-06, 4.562966296361992e-06, 4.273648301023059e-06, 4.03659123549005e-06, 3.775895720536937e-06, 3.474820232440834e-06, 3.1599106478097383e-06, 2.920490715041524e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.6375963190948823e-05, "first_to_last_ratio": 2.3430311680370033, "imbalance_abs_log_ratio": 0.8514454620564724, "last_quartile_mean": 1.5525172557318e-05, "mean": 2.4540472736589436e-05, "normalized": [ 1.7240884069821072, 1.6433567785450998, 1.5754725715380682, 1.4596583982024929, 1.430627916212154, 1.420003858884998, 1.3309607801353691, 1.274107560343628, 1.196376835996058, 1.1165969342919553, 1.0683735188874326, 1.0196491868738, 1.0014614962907418, 0.9675840271884015, 0.9499774532397648, 0.9220954178226054, 0.9056958468427084, 0.8955638114822986, 0.879813549394944, 0.8616003608371102, 0.8461107858930287, 0.8325957474316453, 0.8164485619288154, 0.8006967432790473, 0.7812168094045981, 0.7432274953816075, 0.6968240720487157, 0.659313141540847, 0.6166001053246104, 0.5675655726585476, 0.5174401121065899, 0.47889614301020944 ], "peak_layer": 1, "peak_normalized": 1.7240884069821072, "population_cv": 0.3275253935067736, "rest_mean": 2.5379330947072487e-05, "spike_contrast": 0.7884620142947147, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0010638399980964e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.230994454701431e-05, 4.032875222037546e-05, 3.866284168907441e-05, 3.582070712582208e-05, 3.510828537400812e-05, 3.4847565984819084e-05, 3.266240673838183e-05, 3.126720184809528e-05, 2.9359653126448393e-05, 2.7401816623751074e-05, 2.6218391212751158e-05, 2.5022673071362078e-05, 2.4576338546467014e-05, 2.374496943957638e-05, 2.3312895791605115e-05, 2.262865746160969e-05, 2.2226204237085767e-05, 2.197755929955747e-05, 2.1591040422208607e-05, 2.1144080164958723e-05, 2.076395867334213e-05, 2.0432293240446597e-05, 2.0036033674841747e-05, 1.964947659871541e-05, 1.9171429812558927e-05, 1.823915408749599e-05, 1.710039214231074e-05, 1.6179856174858287e-05, 1.5131658074096777e-05, 1.3928327462053858e-05, 1.2698224963969551e-05, 1.175233774119988e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03225856809876859, "first_to_last_ratio": 2.343031178392381, "imbalance_abs_log_ratio": 0.8514454664761225, "last_quartile_mean": 0.013767878292128444, "mean": 0.02176273672375828, "normalized": [ 1.724088391019206, 1.6433568148707862, 1.575472621674219, 1.4596583934186327, 1.4306278920801987, 1.4200037621599195, 1.3309608349416662, 1.2741075749322037, 1.1963768254907436, 1.1165969105087108, 1.0683735628480904, 1.0196491787212385, 1.001461486526046, 0.9675840094241758, 0.9499774647530201, 0.9220954786760075, 0.9056959063736247, 0.8955638261021951, 0.8798135299584785, 0.8616003323196685, 0.8461108250575042, 0.8325956787797999, 0.8164485932514832, 0.8006966709217642, 0.7812167599757752, 0.7432275143610545, 0.6968240828090287, 0.6593131498508736, 0.6166000906652189, 0.5675655757499506, 0.5174401039988743, 0.478896157779841 ], "peak_layer": 1, "peak_normalized": 1.724088391019206, "population_cv": 0.3275253950778613, "rest_mean": 0.02250664467336955, "spike_contrast": 0.788461987710435, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01774563379585743, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.037520881742239, 0.0357639417052269, 0.03428659588098526, 0.03176616132259369, 0.031134378165006638, 0.0309031680226326, 0.02896535024046898, 0.027728067710995674, 0.02603643387556076, 0.02430020458996296, 0.023250732570886612, 0.022190356627106667, 0.02179454267024994, 0.02105727605521679, 0.02067410945892334, 0.02006732113659382, 0.019710421562194824, 0.019489919766783714, 0.019147150218486786, 0.018750781193375587, 0.018413687124848366, 0.018119560554623604, 0.01776815578341484, 0.017425350844860077, 0.01700141467154026, 0.016174664720892906, 0.015164799056947231, 0.014348458498716354, 0.013418905436992645, 0.012351780198514462, 0.011260912753641605, 0.010422090999782085 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.4063504494288281e-06, "first_to_last_ratio": 1.3888652705487472, "imbalance_abs_log_ratio": 0.32848706162254393, "last_quartile_mean": 1.0125895428814147e-06, "mean": 1.1880723089774392e-06, "normalized": [ 2.22105949166683, 1.3045309379325452, 1.1542973756533603, 1.0756338384718074, 0.9439046493177333, 0.9303001868869332, 0.9181101208153997, 0.9219604060672644, 0.9152949161005884, 0.9117060563851229, 0.9184975703018127, 0.9267480721956175, 0.9157813092197382, 0.9328035373473063, 0.9602268081831139, 0.9736851517288183, 0.975330448461193, 0.9909657433074398, 1.017059016511325, 1.0393447779680196, 1.0410638518191953, 1.0608138238488845, 1.067044114935786, 1.0654680021941936, 1.0596368347940859, 1.0074277055068903, 0.9486012184070485, 0.8859287521128595, 0.8284090099343453, 0.7515767911699255, 0.6962507460711886, 0.6405387346836281 ], "peak_layer": 1, "peak_normalized": 2.22105949166683, "population_cv": 0.2514651034600585, "rest_mean": 1.1751343499276522e-06, "spike_contrast": 1.0704625287514868, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2579372878462892e-06, "top_five_layers": [ 1, 2, 3, 4, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.638779278640868e-06, 1.5498770835620235e-06, 1.3713887483390863e-06, 1.277930778087466e-06, 1.1214269761694595e-06, 1.105263891076902e-06, 1.0907812111327075e-06, 1.0953556284221122e-06, 1.0874365443669376e-06, 1.0831727195181884e-06, 1.0912415291386424e-06, 1.1010437219738378e-06, 1.0880144145630766e-06, 1.1082380524385371e-06, 1.1408188811401487e-06, 1.1568083664315054e-06, 1.1587630979192909e-06, 1.1773389587688143e-06, 1.2083396541129332e-06, 1.234816750184109e-06, 1.2368591342237778e-06, 1.2603235290953307e-06, 1.2677255654125474e-06, 1.2658530295084347e-06, 1.2589251809913549e-06, 1.1968969602094148e-06, 1.1270068398516742e-06, 1.0525474181122263e-06, 9.84209805210412e-07, 8.92927573659108e-07, 8.271962315120618e-07, 7.610063335050654e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6793285996973282e-05, "first_to_last_ratio": 2.59826300786115, "imbalance_abs_log_ratio": 0.954843147867069, "last_quartile_mean": 1.031199917633785e-05, "mean": 1.738467395284715e-05, "normalized": [ 1.78773045454333, 1.7041491227711507, 1.6364518370910566, 1.5208900106186054, 1.4934955268940733, 1.479012186458667, 1.3884863387277648, 1.3193967179334984, 1.2374664756707232, 1.1441290673436744, 1.0866714663117618, 1.0281062332426787, 1.0083205726565403, 0.9674201139660152, 0.9490038741339376, 0.9147691923366695, 0.8949298556467083, 0.8863389586087415, 0.8627560014145367, 0.8374599603804893, 0.8137503987308741, 0.7895173642792594, 0.7624546156174374, 0.7419648877747225, 0.7224315990451545, 0.6853503573208848, 0.6466114342849871, 0.616257335993725, 0.5815823113447253, 0.5363768303272249, 0.49518141464512766, 0.46153748388525473 ], "peak_layer": 1, "peak_normalized": 1.78773045454333, "population_cv": 0.365262728647077, "rest_mean": 1.813793328943171e-05, "spike_contrast": 0.7342111872828369, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3317073535290546e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.107911106781103e-05, 2.9626076866406947e-05, 2.844918162736576e-05, 2.6440176952746697e-05, 2.5963932785089128e-05, 2.5712144633871503e-05, 2.413838228676468e-05, 2.293728175573051e-05, 2.1512951207114384e-05, 1.989031079574488e-05, 1.8891429135692306e-05, 1.7873291653813794e-05, 1.752932439558208e-05, 1.6818283256725408e-05, 1.6498122931807302e-05, 1.5902964150882326e-05, 1.5558063751086593e-05, 1.5408713807119057e-05, 1.4998731785453856e-05, 1.4558968359779101e-05, 1.414678536093561e-05, 1.3725501958106179e-05, 1.3255024896352552e-05, 1.2898817658424377e-05, 1.2559237802634016e-05, 1.1914592505490873e-05, 1.1241128959227353e-05, 1.0713432857301086e-05, 1.0110618859471288e-05, 9.324736311100423e-06, 8.608567441115156e-06, 8.0236786743626e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.938115951816144e-05, "first_to_last_ratio": 2.5763378474631047, "imbalance_abs_log_ratio": 0.946368951607931, "last_quartile_mean": 7.522755424815841e-06, "mean": 1.236969426088308e-05, "normalized": [ 1.8099938946844014, 1.728964663406708, 1.6762891339633759, 1.547604731622059, 1.523697296804946, 1.516312046701088, 1.4092172441217883, 1.322529595442431, 1.2410675898615309, 1.1253325211094913, 1.0636520363082502, 1.00702477602901, 0.9880485860245845, 0.9448364468322119, 0.9350182198861665, 0.8997048366987048, 0.8789956378042715, 0.8856125407337307, 0.8462418433091708, 0.8155022213016762, 0.7864781089853847, 0.750887679658867, 0.7217551899516604, 0.7099517597798954, 0.7012053955236477, 0.6724904602282508, 0.6442506502382093, 0.6188711504662027, 0.5967590010995487, 0.5664323381260263, 0.5432338046756088, 0.522038598621101 ], "peak_layer": 1, "peak_normalized": 1.8099938946844014, "population_cv": 0.37345755309966855, "rest_mean": 1.2978888814056214e-05, "spike_contrast": 0.699601006205894, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.08004367374815e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.2389071091311052e-05, 2.13867642742116e-05, 2.073518407996744e-05, 1.914339736686088e-05, 1.8847669707611203e-05, 1.8756316421786323e-05, 1.7431586456950754e-05, 1.635928674659226e-05, 1.5351626643678173e-05, 1.3920019227953162e-05, 1.3157050489098765e-05, 1.2456588592613116e-05, 1.2221858924021944e-05, 1.1687337973853573e-05, 1.1565889508347027e-05, 1.1129073755000718e-05, 1.0872907296288759e-05, 1.0954756362480111e-05, 1.0467752872500569e-05, 1.0087513146572746e-05, 9.72849375102669e-06, 9.2882510216441e-06, 8.927891030907631e-06, 8.781886208453216e-06, 8.673696356709115e-06, 8.318501386384014e-06, 7.96918357082177e-06, 7.655246918147895e-06, 7.381726391031407e-06, 7.006594842096092e-06, 6.719636076013558e-06, 6.457457857322879e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.503894994442817e-05, "first_to_last_ratio": 2.2837759148871593, "imbalance_abs_log_ratio": 0.8258301761605996, "last_quartile_mean": 3.285740490355238e-05, "mean": 5.1171222082757595e-05, "normalized": [ 1.7250540829663195, 1.6170039295082501, 1.558641962352184, 1.4475704461622705, 1.405871545434971, 1.379249740977325, 1.3192988177043423, 1.2787392731618172, 1.2121135966900498, 1.1204005426005206, 1.0730027281223469, 1.0188897789208062, 0.9948840289702825, 0.9615972823180635, 0.9476620305313541, 0.9234741371548107, 0.9102494722311251, 0.908601578994704, 0.8796202280241118, 0.8553643685604198, 0.8418081912263009, 0.8360712420605473, 0.8303072059918611, 0.8176670067611138, 0.7905207401435352, 0.7558412574586363, 0.7089931435932589, 0.6694146283776838, 0.6276662460639247, 0.5712927135223572, 0.5255955797489182, 0.48753247366578784 ], "peak_layer": 1, "peak_normalized": 1.7250540829663195, "population_cv": 0.3189635247096411, "rest_mean": 5.2845896250262204e-05, "spike_contrast": 0.7971854877572201, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.212798157823272e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.827312558423728e-05, 8.274406718555838e-05, 7.975761400302872e-05, 7.407394878100604e-05, 7.194016507128254e-05, 7.057789480313659e-05, 6.751013279426843e-05, 6.543465133290738e-05, 6.202533404575661e-05, 5.733226498705335e-05, 5.4906860896153376e-05, 5.213783515500836e-05, 5.0909431593026966e-05, 4.920610808767378e-05, 4.849302422371693e-05, 4.7255300160031766e-05, 4.6578577894251794e-05, 4.6494253183482215e-05, 4.50112420367077e-05, 4.3770040065282956e-05, 4.3076353904325515e-05, 4.278278720448725e-05, 4.248783443472348e-05, 4.184101999271661e-05, 4.0451912354910746e-05, 3.867732084472664e-05, 3.6280045605963096e-05, 3.4254764614161104e-05, 3.2118448871187866e-05, 2.9233746317913756e-05, 2.6895368137047626e-05, 2.4947632482508197e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13547764718532562, 0.039667773991823196, 0.026703152805566788, 0.05304057523608208, 0.016798952594399452, 0.00753376679494977, 0.01294450368732214, 0.03526800498366356, 0.030294200405478477, 0.034149833023548126, 0.04367009550333023, 0.05930330231785774, 0.04527241364121437, 0.04281846433877945, 0.04812675341963768, 0.06094835698604584, 0.038031257688999176, 0.040106989443302155, 0.04961257055401802, 0.07085621356964111, 0.03753058984875679, 0.04544757679104805, 0.05368982255458832, 0.06174425780773163, 0.05051252990961075, 0.054937634617090225, 0.05777662619948387, 0.07133159041404724, 0.0636468306183815, 0.09261580556631088, 0.1020733118057251, 0.12878532707691193 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.744960837248072e-06, "first_to_last_ratio": 2.4076765961345377, "imbalance_abs_log_ratio": 0.8786622144961542, "last_quartile_mean": 3.6321160621355375e-06, "mean": 5.9219457781978235e-06, "normalized": [ 1.5966609800930127, 1.6382818782556905, 1.6078801806645238, 1.4395757389908588, 1.4506416663399584, 1.4388820164000453, 1.348677960564097, 1.29303133390772, 1.1804814091096145, 1.1203943128344216, 1.0731688825386334, 1.0349744395148408, 1.011916879057038, 0.9796669183445198, 0.9673471607471578, 0.9423088740875128, 0.9226627178910438, 0.9142650193290811, 0.8983732123895815, 0.882245122030563, 0.8656871601790304, 0.8478113250594501, 0.8280201889812614, 0.8103923861805131, 0.7714582232411454, 0.7214453089906826, 0.6827625571385767, 0.6375172771181937, 0.58753815055203, 0.5337410156600697, 0.4936733910977551, 0.4785163127113774 ], "peak_layer": 2, "peak_normalized": 1.6382818782556905, "population_cv": 0.32643117815601047, "rest_mean": 6.114218354856281e-06, "spike_contrast": 0.798740506276366, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.883673864242155e-06, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 9.455339750275016e-06, 9.701816452434286e-06, 9.52177924773423e-06, 8.525089469912928e-06, 8.590621291659772e-06, 8.52098128234502e-06, 7.986797754711006e-06, 7.657261448912323e-06, 6.990746896917699e-06, 6.634914370806655e-06, 6.355247933242936e-06, 6.12906251262757e-06, 5.992516889818944e-06, 5.801534371130401e-06, 5.728577434638282e-06, 5.580302058660891e-06, 5.463958586915396e-06, 5.414227871369803e-06, 5.320117452356499e-06, 5.224607775744516e-06, 5.126552423462272e-06, 5.0206926971441135e-06, 4.903490662400145e-06, 4.79909977002535e-06, 4.568533768178895e-06, 4.272360001777997e-06, 4.0432828427583445e-06, 3.775342747758259e-06, 3.4793690701917512e-06, 3.1607853543391684e-06, 2.9235070542199537e-06, 2.8337476578599308e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.557536911102943e-05, "first_to_last_ratio": 2.4432305702295483, "imbalance_abs_log_ratio": 0.8933211677768842, "last_quartile_mean": 1.4560790759787778e-05, "mean": 2.3831157136555703e-05, "normalized": [ 1.614913653123857, 1.6568181419735588, 1.628341463280827, 1.4605653558450646, 1.471807278919832, 1.4559143731134858, 1.3583919651208736, 1.295720862036415, 1.177691966325428, 1.114101795346675, 1.0640412563479125, 1.026954832666515, 1.0041441491510337, 0.9733761741275299, 0.9597031214537947, 0.934968890890384, 0.9158576063976391, 0.906677613403137, 0.8903545981719634, 0.8734940797341108, 0.8579208364426144, 0.8414682922506719, 0.8239516565784237, 0.8048355634088795, 0.765892761498337, 0.717371762357387, 0.6794952654450417, 0.6347194240846251, 0.5854549882448604, 0.5328880600388105, 0.4936169323623591, 0.4785452798579532 ], "peak_layer": 2, "peak_normalized": 1.6568181419735588, "population_cv": 0.333834364988221, "rest_mean": 2.463076385471073e-05, "spike_contrast": 0.7922320628633921, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9513280858518555e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.8485261029563844e-05, 3.9483893488068134e-05, 3.880526128341444e-05, 3.4806962503353134e-05, 3.507487053866498e-05, 3.469612420303747e-05, 3.237205237383023e-05, 3.087852746830322e-05, 2.806576230796054e-05, 2.6550334951025434e-05, 2.5357334379805252e-05, 2.4473521989420988e-05, 2.392991700617131e-05, 2.319668055861257e-05, 2.2870835891808383e-05, 2.2281390556599945e-05, 2.1825946532771923e-05, 2.1607176677207462e-05, 2.121818033629097e-05, 2.081637467199471e-05, 2.0445246263989247e-05, 2.0053163098054938e-05, 1.9635721400845796e-05, 1.918016278068535e-05, 1.8252110749017447e-05, 1.7095799194066785e-05, 1.6193158444366418e-05, 1.5126098332984839e-05, 1.395206982124364e-05, 1.2699339094979223e-05, 1.1763462680391967e-05, 1.1404287761251908e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03154859389178455, "first_to_last_ratio": 2.443230668285183, "imbalance_abs_log_ratio": 0.8933212079104819, "last_quartile_mean": 0.012912654667161405, "mean": 0.021133708796696737, "normalized": [ 1.6149137386398011, 1.656818103118424, 1.6283414676061427, 1.4605652763644328, 1.471807315279864, 1.4559144163177982, 1.3583919563693225, 1.2957209363886424, 1.1776919990513512, 1.1141018064612538, 1.0640412283046075, 1.0269548301404738, 1.0041441194885832, 0.9733762067371586, 0.9597031065420684, 0.9349689110855055, 0.9158576300153286, 0.9066776260380441, 0.8903546223359304, 0.8734940758449745, 0.857920846942776, 0.8414682830013448, 0.8239516505093816, 0.8048355219472574, 0.7658927106397581, 0.7173717399936698, 0.6794952447102416, 0.634719402274497, 0.5854550026361336, 0.5328880508022114, 0.4936169320855568, 0.47854524232746515 ], "peak_layer": 2, "peak_normalized": 1.656818103118424, "population_cv": 0.3338343750142194, "rest_mean": 0.0218428081177451, "spike_contrast": 0.7922320413087064, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017304572463035583, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.03412911668419838, 0.03501471132040024, 0.034412894397974014, 0.030867161229252815, 0.03110474720597267, 0.030768871307373047, 0.02870786003768444, 0.027383388951420784, 0.02488899976015091, 0.023545103147625923, 0.022487137466669083, 0.021703364327549934, 0.021221289411187172, 0.02057104930281639, 0.020282085984945297, 0.01975936070084572, 0.019355468451976776, 0.019161460921168327, 0.018816495314240456, 0.018460169434547424, 0.01813104934990406, 0.01778334565460682, 0.017413154244422913, 0.017009159550070763, 0.016186153516173363, 0.015160725452005863, 0.014360254630446434, 0.013413975015282631, 0.012372835539281368, 0.011261900886893272, 0.010431956499814987, 0.010113435797393322 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.344927397894935e-06, "first_to_last_ratio": 1.4244585458149095, "imbalance_abs_log_ratio": 0.3537917736653531, "last_quartile_mean": 9.441674535537459e-07, "mean": 1.1411028104646448e-06, "normalized": [ 2.099538361004741, 1.2695654171364545, 1.1994886284285255, 1.0098034387773935, 0.9820379553982943, 0.96728171717488, 0.9479405561697765, 0.9533095585319065, 0.9297712304918835, 0.9212055346840066, 0.9197232556499785, 0.928745849673345, 0.9294609860429008, 0.9490990412220468, 0.9764042390024628, 0.987029961936741, 0.9894569223661618, 1.0074939408416628, 1.0338473835735458, 1.0587716494844093, 1.0614411066989373, 1.080242378600304, 1.0792969998024888, 1.0997107657330087, 1.050181644220671, 0.9776416301573645, 0.927097788120657, 0.865770311315491, 0.7904398407173965, 0.71069112651653, 0.6594538672142737, 0.6380569133117612 ], "peak_layer": 1, "peak_normalized": 2.099538361004741, "population_cv": 0.23657254437119296, "rest_mean": 1.1254285844323156e-06, "spike_contrast": 1.0891349731067184, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2257436310392222e-06, "top_five_layers": [ 1, 2, 3, 24, 22 ], "top_five_spike_overlap": 2 }, "values": [ 2.395789124420844e-06, 1.4487046655631275e-06, 1.3687398450201727e-06, 1.1522895420057466e-06, 1.1206062708879472e-06, 1.1037678859793232e-06, 1.0816976327987504e-06, 1.0878242164835683e-06, 1.0609645642034593e-06, 1.0511902246435056e-06, 1.0494987918718834e-06, 1.0597944992696284e-06, 1.0606105433907942e-06, 1.0830195833477774e-06, 1.114177621275303e-06, 1.1263026635788265e-06, 1.129072074945725e-06, 1.1496541674205218e-06, 1.1797261549872928e-06, 1.2081673048669472e-06, 1.2112134299968602e-06, 1.2326676142038195e-06, 1.2315888398006791e-06, 1.2548830454761628e-06, 1.1983652257185895e-06, 1.1155896117998054e-06, 1.0579138916000375e-06, 9.879329354589572e-07, 9.019731237458473e-07, 8.109716418402968e-07, 7.525046612499864e-07, 7.280885370164469e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.630348421917006e-05, "first_to_last_ratio": 2.69255008587913, "imbalance_abs_log_ratio": 0.9904887318732114, "last_quartile_mean": 9.768986046765349e-06, "mean": 1.6957483779833638e-05, "normalized": [ 1.6937424339749063, 1.716030827709739, 1.683456753729431, 1.520726904446963, 1.5301933816635431, 1.5109070875973878, 1.4130243932548368, 1.3410631406221931, 1.2180743459632961, 1.139878912155672, 1.078689369482256, 1.0334527787088414, 1.0088730407419506, 0.9710503607378324, 0.9566665199949184, 0.9261587427208572, 0.9027566962629673, 0.8950285985522498, 0.8707223394543085, 0.846760516774566, 0.8234152149025455, 0.7960865825504274, 0.7684196280360551, 0.7461253882143986, 0.7082562076848088, 0.6648380180827745, 0.633671939252038, 0.597754010876058, 0.5540441605387132, 0.5092663534964215, 0.477652324967025, 0.4632130268500192 ], "peak_layer": 2, "peak_normalized": 1.716030827709739, "population_cv": 0.37037219142929884, "rest_mean": 1.7684581103417764e-05, "spike_contrast": 0.7368655302760281, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3031158232479356e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.872160985134542e-05, 2.909956492658239e-05, 2.8547190595418215e-05, 2.578770181571599e-05, 2.5948229449568316e-05, 2.5621182430768386e-05, 2.396133822912816e-05, 2.2741056454833597e-05, 2.065547596430406e-05, 1.932947816385422e-05, 1.8291857486474328e-05, 1.752475873217918e-05, 1.7107948224293068e-05, 1.6466570741613396e-05, 1.622265699552372e-05, 1.570532185724005e-05, 1.530848203401547e-05, 1.5177432942437008e-05, 1.4765259948035236e-05, 1.435892772860825e-05, 1.3963050150778145e-05, 1.3499625310942065e-05, 1.3030463378527202e-05, 1.265240916836774e-05, 1.201024315378163e-05, 1.1273979907855392e-05, 1.074548163160216e-05, 1.0136403943761252e-05, 9.395194865646772e-06, 8.63587592903059e-06, 8.099781553028151e-06, 7.854927389416844e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.907835189740581e-05, "first_to_last_ratio": 2.6287461590420254, "imbalance_abs_log_ratio": 0.9665069868989548, "last_quartile_mean": 7.257586219111545e-06, "mean": 1.212781258175255e-05, "normalized": [ 1.7354359984845942, 1.7341266298424, 1.7101923309667906, 1.5428934146424107, 1.5519006110270992, 1.5422286813294432, 1.4249678951055762, 1.34311375683423, 1.2181905393168442, 1.117207245110816, 1.0512122908225914, 1.0107192781417305, 0.9868283999100976, 0.9491456552989723, 0.9450899871696192, 0.910761731031231, 0.8806493269523064, 0.8929657163538394, 0.8505844681035736, 0.8211947665220615, 0.7893804081857143, 0.7583735688378634, 0.7282248684002433, 0.7172124481825861, 0.6937135556263685, 0.6567877100820053, 0.6342047756355106, 0.6086855107660424, 0.5784802660010331, 0.5544185549076066, 0.5338091599726986, 0.5273004504361006 ], "peak_layer": 1, "peak_normalized": 1.7354359984845942, "population_cv": 0.3752406439924748, "rest_mean": 1.2717626355475255e-05, "spike_contrast": 0.7031829646260871, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.942818203649949e-06, "top_five_layers": [ 1, 2, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.1047042537247762e-05, 2.1031162759754807e-05, 2.0740892068715766e-05, 1.8711922166403383e-05, 1.8821159756043926e-05, 1.8703860405366868e-05, 1.7281743566854857e-05, 1.628903191885911e-05, 1.4773986549698748e-05, 1.354928008368006e-05, 1.2748905646731146e-05, 1.2257813978067134e-05, 1.196806988446042e-05, 1.1511060620250646e-05, 1.1461874237284064e-05, 1.1045547580579296e-05, 1.0680349987524096e-05, 1.0829720849869773e-05, 1.031572901410982e-05, 9.959296221495606e-06, 9.57345764618367e-06, 9.197412509820424e-06, 8.831774721329566e-06, 8.698218152858317e-06, 8.41322798805777e-06, 7.96539825387299e-06, 7.691516657359898e-06, 7.3820237957988866e-06, 7.015700248302892e-06, 6.723884325765539e-06, 6.473937446571654e-06, 6.395001037162729e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.32765902284882e-05, "first_to_last_ratio": 2.3797059401792184, "imbalance_abs_log_ratio": 0.8669769255033447, "last_quartile_mean": 3.079228781643906e-05, "mean": 4.9631882802714244e-05, "normalized": [ 1.6036295653160026, 1.6264177096181267, 1.605845107850632, 1.4451626399742143, 1.462521070493661, 1.4193768503503077, 1.3493265351473542, 1.2989331677271607, 1.1827368823021753, 1.1159788020275245, 1.069537069411288, 1.0194230453577149, 0.999117032890771, 0.9700459717654388, 0.9607374093144786, 0.9410261468020766, 0.9223083820569733, 0.9136749788161789, 0.8935785288486232, 0.8650748037657058, 0.859486397149459, 0.8548828389913368, 0.8395424095808984, 0.8183290259422422, 0.7781609011469676, 0.7313438031724356, 0.6926703962811562, 0.6467311298885312, 0.5885390175628117, 0.5417837841386413, 0.5025152241470534, 0.48156337216205825 ], "peak_layer": 2, "peak_normalized": 1.6264177096181267, "population_cv": 0.32403192338043996, "rest_mean": 5.119362982373719e-05, "spike_contrast": 0.80475733076633, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.119844888919033e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.959115464473143e-05, 8.072217315202579e-05, 7.970111619215459e-05, 7.172614277806133e-05, 7.258767436724156e-05, 7.044634548947215e-05, 6.696961645502597e-05, 6.44684987491928e-05, 5.8701458328869194e-05, 5.538812911254354e-05, 5.308313848217949e-05, 5.059588511358015e-05, 4.958805948263034e-05, 4.814520798390731e-05, 4.768320650327951e-05, 4.670489943237044e-05, 4.577590152621269e-05, 4.534740946837701e-05, 4.4349984818836674e-05, 4.2935291276080534e-05, 4.2657928133849055e-05, 4.242944487486966e-05, 4.166807048022747e-05, 4.061521030962467e-05, 3.86215906473808e-05, 3.629796992754564e-05, 3.437853592913598e-05, 3.2098483643494546e-05, 2.921029954450205e-05, 2.6889749278780073e-05, 2.4940776711446233e-05, 2.3900896849227138e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08626964688301086, 0.27676042914390564, 0.41335058212280273, 0.43042075634002686, 0.1360139101743698, 0.15445689857006073, 0.1670466512441635, 0.17834730446338654, 0.05234000086784363, 0.08269349485635757, 0.11548293381929398, 0.16410532593727112, 0.062187470495700836, 0.0988159254193306, 0.13199056684970856, 0.17199495434761047, 0.04149825870990753, 0.06434677541255951, 0.08728200942277908, 0.11647984385490417, 0.038355328142642975, 0.06353354454040527, 0.09196189045906067, 0.1169828325510025, 0.027557097375392914, 0.06579430401325226, 0.09665275365114212, 0.12799666821956635, 0.019497457891702652, 0.07176875323057175, 0.14131556451320648, 0.22250843048095703 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.943195439314877e-06, "first_to_last_ratio": 2.307887867651198, "imbalance_abs_log_ratio": 0.8363327632590157, "last_quartile_mean": 3.875056307833802e-06, "mean": 6.099261987912996e-06, "normalized": [ 1.6993364194524787, 1.6239631401407362, 1.5555352150060116, 1.4421180897792774, 1.410443415622084, 1.403515806193484, 1.322136600562147, 1.2731514024188308, 1.2000780810869136, 1.123528951240296, 1.07670935798164, 1.026140851109733, 1.0093877955434476, 0.9743169405661355, 0.9573130726734715, 0.928737466637045, 0.9116867764670428, 0.9026819876648067, 0.886529122118542, 0.8702279611747841, 0.8533837960178821, 0.8403751805988604, 0.8216892894948097, 0.8043572928719456, 0.7863148352924635, 0.748123886904242, 0.7006953058091231, 0.6618269255548263, 0.619067487305518, 0.5697120352939504, 0.5180849945326964, 0.47883051688477524 ], "peak_layer": 1, "peak_normalized": 1.6993364194524787, "population_cv": 0.31980786012147844, "rest_mean": 6.301188132393118e-06, "spike_contrast": 0.7949073575459225, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.008860807720339e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.0364698027842678e-05, 9.904976650432218e-06, 9.487616807746235e-06, 8.795856047072448e-06, 8.602663911005948e-06, 8.560410606150981e-06, 8.064057510637213e-06, 7.765283953631297e-06, 7.319590622500982e-06, 6.8526974246196914e-06, 6.567132459167624e-06, 6.258701887418283e-06, 6.1565206124214455e-06, 5.942614279774716e-06, 5.8389032346894965e-06, 5.664613127009943e-06, 5.560616500588367e-06, 5.505693934537703e-06, 5.407173375715502e-06, 5.307748324412387e-06, 5.205011348152766e-06, 5.125668394612148e-06, 5.01169824929093e-06, 4.905985861114459e-06, 4.795940185431391e-06, 4.563003585644765e-06, 4.273724243830657e-06, 4.036655809613876e-06, 3.7758547932753572e-06, 3.4748229609249393e-06, 3.159936113661388e-06, 2.920512770288042e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.637614054241567e-05, "first_to_last_ratio": 2.3430282822498043, "imbalance_abs_log_ratio": 0.8514442304087609, "last_quartile_mean": 1.552526737214066e-05, "mean": 2.454050053302126e-05, "normalized": [ 1.7240897155140114, 1.6433731511507879, 1.5755074032746206, 1.4596353977652112, 1.4306148809963868, 1.42000847672355, 1.3309924792154737, 1.2740991497308294, 1.1963637696230847, 1.1165890726973144, 1.0683694180110759, 1.019625350629474, 1.001471406127935, 0.96757892864483, 0.9499723005167983, 0.9220974865087133, 0.905693931519684, 0.8955733965356255, 0.8798087726326785, 0.8615996072890131, 0.8461088639386269, 0.8325951008583086, 0.8164437828166039, 0.8006795295233357, 0.7812208907088943, 0.743232064447096, 0.6968341045755492, 0.659316323215381, 0.6166001110749636, 0.567565559827195, 0.5174384883075808, 0.4789010855993677 ], "peak_layer": 1, "peak_normalized": 1.7240897155140114, "population_cv": 0.32752712068205064, "rest_mean": 2.5379378308166957e-05, "spike_contrast": 0.7884574753667308, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.001056054723449e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.231002458254807e-05, 4.0329199691768736e-05, 3.8663740269839764e-05, 3.582018325687386e-05, 3.510800524963997e-05, 3.484771877992898e-05, 3.266322164563462e-05, 3.126703086309135e-05, 2.9359365726122633e-05, 2.740165473369416e-05, 2.621832027216442e-05, 2.5022116460604593e-05, 2.4576609575888142e-05, 2.374487121414859e-05, 2.331279574718792e-05, 2.262873385916464e-05, 2.2226182409212925e-05, 2.1977819415042177e-05, 2.1590947653749026e-05, 2.1144085621926934e-05, 2.0763935026479885e-05, 2.043230051640421e-05, 2.0035939087392762e-05, 1.964907642104663e-05, 1.9171551684848964e-05, 1.823928687372245e-05, 1.7100657714763656e-05, 1.6179952581296675e-05, 1.513167535449611e-05, 1.3928342923463788e-05, 1.2698199498117901e-05, 1.1752472346415743e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03225872525945306, "first_to_last_ratio": 2.3430283290295004, "imbalance_abs_log_ratio": 0.8514442503742458, "last_quartile_mean": 0.013767962111160159, "mean": 0.021762761170975864, "normalized": [ 1.7240897066312928, 1.6433731135962664, 1.5755074838038876, 1.4596353565498406, 1.430614987282464, 1.4200084149681802, 1.33099254819824, 1.2740991253937513, 1.1963637558959541, 1.116589065853211, 1.0683694526758873, 1.019625352307586, 1.0014714024680393, 0.9675788998216788, 0.9499723749288511, 0.9220975240302813, 0.9056938618956135, 0.8955734330612176, 0.8798086901303112, 0.8615996212060729, 0.8461089331030892, 0.8325950858372491, 0.8164437390157344, 0.8006795096166728, 0.7812208465363841, 0.7432320715401213, 0.6968340841998046, 0.6593163034940038, 0.6166001255103924, 0.5675655372951299, 0.5174384956683723, 0.4789010974844188 ], "peak_layer": 1, "peak_normalized": 1.7240897066312928, "population_cv": 0.327527126007556, "rest_mean": 0.022506686410418263, "spike_contrast": 0.788457463457284, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017745564877986907, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.03752095252275467, 0.035764336585998535, 0.03428739309310913, 0.03176569566130638, 0.03113413229584694, 0.030903303995728493, 0.02896607294678688, 0.027727914974093437, 0.026036178693175316, 0.02430006116628647, 0.023250669240951538, 0.02218986302614212, 0.02179478295147419, 0.021057188510894775, 0.020674021914601326, 0.02006738819181919, 0.019710399210453033, 0.01949015073478222, 0.01914706639945507, 0.018750786781311035, 0.018413666635751724, 0.0181195680052042, 0.017768070101737976, 0.017424996942281723, 0.017001522704958916, 0.016174782067537308, 0.015165033750236034, 0.014348543249070644, 0.013418921269476414, 0.012351793237030506, 0.011260890401899815, 0.010422210209071636 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.4063588764656743e-06, "first_to_last_ratio": 1.3888550856996456, "imbalance_abs_log_ratio": 0.3284797283795997, "last_quartile_mean": 1.0126030360879668e-06, "mean": 1.1880729502422582e-06, "normalized": [ 2.221048723833859, 1.3045184639250926, 1.1544035427877724, 1.075575269686293, 0.9438353386502788, 0.9303903032947847, 0.9181045536874566, 0.9219724438423569, 0.9151919379546727, 0.911597530149608, 0.918441095823346, 0.9266988657126964, 0.9158950689226896, 0.9328019812734085, 0.9602261942086033, 0.9737021374713742, 0.9753680066905094, 0.9909324824121717, 1.0171062169192169, 1.0393085245664904, 1.041121852253469, 1.0608184185386729, 1.0669551213298931, 1.0655289558498546, 1.0596357844023474, 1.0074707964383398, 0.9486502738773706, 0.8859198532008371, 0.8284323896376146, 0.751570117802247, 0.6962432413548182, 0.6405345135018543 ], "peak_layer": 1, "peak_normalized": 2.221048723833859, "population_cv": 0.25146614998929095, "rest_mean": 1.17513350990824e-06, "spike_contrast": 1.0704706464750395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.257945928045956e-06, "top_five_layers": [ 1, 2, 3, 4, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.638767909957096e-06, 1.5498631000809837e-06, 1.3715156228499836e-06, 1.2778618838638067e-06, 1.1213452353331377e-06, 1.1053715525122243e-06, 1.0907751857303083e-06, 1.0953705213978537e-06, 1.0873147857637377e-06, 1.0830443670784007e-06, 1.0911750223385752e-06, 1.1009858553734375e-06, 1.0881501566473162e-06, 1.1082368018833222e-06, 1.140818767453311e-06, 1.1568291711228085e-06, 1.158808345280704e-06, 1.1773000778703135e-06, 1.2083963838449563e-06, 1.2347743449936388e-06, 1.2369287105684634e-06, 1.2603296681845677e-06, 1.2676205187744927e-06, 1.2659261301450897e-06, 1.2589246125571663e-06, 1.1969488014074159e-06, 1.127065729633614e-06, 1.0525374136705068e-06, 9.842381132330047e-07, 8.929201271712373e-07, 8.271877618426515e-07, 7.610017291881377e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6793339202413335e-05, "first_to_last_ratio": 2.5982590296524397, "imbalance_abs_log_ratio": 0.9548416167627336, "last_quartile_mean": 1.0312035442439083e-05, "mean": 1.738467847189895e-05, "normalized": [ 1.7877226656098162, 1.7041567364320032, 1.6364649092006618, 1.5208911847479025, 1.4934925228742508, 1.4790155687407116, 1.3884983243453457, 1.3193915619028695, 1.2374632243089856, 1.1441214457108106, 1.0866704514151964, 1.028096130607215, 1.0083233448700089, 0.9674170374331021, 0.9490011162834874, 0.9147690591791905, 0.8949296230146019, 0.8863450061150394, 0.8627491853453367, 0.8374612075319247, 0.813747832986454, 0.7895165835739041, 0.7624542081580346, 0.7419568475234603, 0.7224344978897704, 0.6853530565410577, 0.6466163408419674, 0.6162592684360938, 0.5815819509020856, 0.5363784696388306, 0.49517950718561515, 0.4615411306542665 ], "peak_layer": 1, "peak_normalized": 1.7877226656098162, "population_cv": 0.3652632831630456, "rest_mean": 1.8137943731778284e-05, "spike_contrast": 0.7342092502590932, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3317046068550553e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.107898373855278e-05, 2.962621692859102e-05, 2.8449416276998818e-05, 2.644020423758775e-05, 2.596388731035404e-05, 2.571221011749003e-05, 2.41385969275143e-05, 2.2937198082217947e-05, 2.1512900275411084e-05, 1.9890183466486633e-05, 1.889141640276648e-05, 1.787312066880986e-05, 1.7529377146274783e-05, 1.6818234144011512e-05, 1.6498079276061617e-05, 1.590296596987173e-05, 1.5558063751086593e-05, 1.540882294648327e-05, 1.4998617189121433e-05, 1.455899382563075e-05, 1.414674443367403e-05, 1.372549195366446e-05, 1.3255021258373745e-05, 1.2898681234219112e-05, 1.255929146282142e-05, 1.191464252769947e-05, 1.1241217180213425e-05, 1.0713469237089157e-05, 1.011061522149248e-05, 9.324767233920284e-06, 8.608536518295296e-06, 8.023744157981128e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.938144146151899e-05, "first_to_last_ratio": 2.5765408114136847, "imbalance_abs_log_ratio": 0.9464477285262795, "last_quartile_mean": 7.522272255755524e-06, "mean": 1.2369578129778347e-05, "normalized": [ 1.8100777970421271, 1.7287935495739235, 1.6764760419525127, 1.547606614625523, 1.5236827794729195, 1.516550097896885, 1.4092148868125007, 1.322506866159222, 1.2411937962106272, 1.1252563246839273, 1.0634665882834902, 1.0069909967052106, 0.9883696891505255, 0.9448880364107244, 0.9350719230888654, 0.899703210365348, 0.8790680055200518, 0.8856083557133494, 0.8461178812403954, 0.8154512032643977, 0.7864210833714382, 0.7508010562653247, 0.7218124054512957, 0.7098562229603778, 0.7011466870106873, 0.6724037625473418, 0.6441132480887107, 0.6188962247087517, 0.5967934262190382, 0.5664043851970144, 0.5432476544778881, 0.5220091995296049 ], "peak_layer": 1, "peak_normalized": 1.8100777970421271, "population_cv": 0.37348547055792297, "rest_mean": 1.2978877226420014e-05, "spike_contrast": 0.6995491866916842, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.079363007913343e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.2389898731489666e-05, 2.1384446881711483e-05, 2.0737301383633167e-05, 1.9143240933772177e-05, 1.8847313185688108e-05, 1.875908492365852e-05, 1.7431393644073978e-05, 1.6358852008124813e-05, 1.5353043636423536e-05, 1.391894602420507e-05, 1.3154633052181453e-05, 1.2456053809728473e-05, 1.2225716091052163e-05, 1.1687866390275303e-05, 1.156644520960981e-05, 1.1128949154226575e-05, 1.0873700375668705e-05, 1.095460174838081e-05, 1.0466121239005588e-05, 1.0086787369800732e-05, 9.727697033667937e-06, 9.287092325394042e-06, 8.928514944273047e-06, 8.780622010817751e-06, 8.67288872541394e-06, 8.317350875586271e-06, 7.96740914665861e-06, 7.65548520575976e-06, 7.382082912954502e-06, 7.00618329574354e-06, 6.719744305883069e-06, 6.457033578044502e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.505454232159536e-05, "first_to_last_ratio": 2.2842016822178994, "imbalance_abs_log_ratio": 0.826016590092333, "last_quartile_mean": 3.2858106578714796e-05, "mean": 5.1174989891933365e-05, "normalized": [ 1.7246853715589219, 1.6173583287304225, 1.5597620217115067, 1.4476516844938752, 1.4063499712999827, 1.3795411723395463, 1.3191459492505877, 1.2785090603756792, 1.2120772438445848, 1.1201281732875392, 1.0727436588021377, 1.0187485782963843, 0.994689288551661, 0.9615888999095473, 0.9476802661591673, 0.9234563342926768, 0.910357048796321, 0.9086057002160716, 0.879517219234551, 0.8554380245700975, 0.8417689607464686, 0.835900421698054, 0.830223112057046, 0.8174852430317888, 0.7904688641941676, 0.7556805383589447, 0.7090539036922652, 0.6694271894936573, 0.6276488245612287, 0.5712677128918332, 0.5256154951349393, 0.4874257384183444 ], "peak_layer": 1, "peak_normalized": 1.7246853715589219, "population_cv": 0.31905266441255564, "rest_mean": 5.285078771487827e-05, "spike_contrast": 0.7970681889415223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2125681648030874e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.826075645629317e-05, 8.276829612441361e-05, 7.98208056949079e-05, 7.408356032101437e-05, 7.196994556579739e-05, 7.059800554998219e-05, 6.750728061888367e-05, 6.54276882414706e-05, 6.202804070198908e-05, 5.732254794565961e-05, 5.489764589583501e-05, 5.213444819673896e-05, 5.0903214287245646e-05, 4.920930223306641e-05, 4.849752804148011e-05, 4.725786857306957e-05, 4.658751277020201e-05, 4.64978875243105e-05, 4.5009284804109484e-05, 4.3777032260550186e-05, 4.3077518057543784e-05, 4.277719563106075e-05, 4.248665936756879e-05, 4.1834799048956484e-05, 4.045223613502458e-05, 3.8671943912049755e-05, 3.6285826354287565e-05, 3.4257929655723274e-05, 3.211992225260474e-05, 2.9234619432827458e-05, 2.6898367650574073e-05, 2.494400723662693e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16424456238746643, 0.29588937759399414, 0.42337876558303833, 0.45154979825019836, 0.13891471922397614, 0.1555756777524948, 0.16914354264736176, 0.1874145120382309, 0.06211879849433899, 0.09842939674854279, 0.137006014585495, 0.20118924975395203, 0.08399304747581482, 0.12138351052999496, 0.1587214469909668, 0.2071460783481598, 0.0581100657582283, 0.08358953893184662, 0.10947012156248093, 0.1472771316766739, 0.05584336072206497, 0.08318652212619781, 0.11238116025924683, 0.1379220336675644, 0.05995604768395424, 0.09392217546701431, 0.1259489357471466, 0.17086055874824524, 0.06700707972049713, 0.13971327245235443, 0.2211977243423462, 0.32473331689834595 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.744833621676662e-06, "first_to_last_ratio": 2.407641608620192, "imbalance_abs_log_ratio": 0.8786476827402625, "last_quartile_mean": 3.6321160052921186e-06, "mean": 5.921910066319924e-06, "normalized": [ 1.5966143979437075, 1.6382704100570205, 1.6078846551669277, 1.4395458713949063, 1.4506247663066902, 1.438881478650706, 1.3486619814552512, 1.2930475784613584, 1.1804812328505396, 1.1204054464016595, 1.0731833404752842, 1.034984213263227, 1.0119477847689309, 0.9796848055347882, 0.9673465438864248, 0.9423091045127517, 0.9226633673753865, 0.9142662324966246, 0.8983748672567047, 0.882237157596605, 0.8656713400367232, 0.8478121374802432, 0.8280229553999554, 0.8103965821084333, 0.7714594199143477, 0.7214453593654997, 0.6827608384252767, 0.6375225038763053, 0.5875414633089651, 0.533745309436336, 0.49368765640464074, 0.47851919838777923 ], "peak_layer": 2, "peak_normalized": 1.6382704100570205, "population_cv": 0.326425065508555, "rest_mean": 6.114182985617879e-06, "spike_contrast": 0.798738983376607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.883636302110972e-06, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 9.455006875214167e-06, 9.70169003267074e-06, 9.52174832491437e-06, 8.524861186742783e-06, 8.590469406044576e-06, 8.520926712662913e-06, 7.986654964042827e-06, 7.65731147112092e-06, 6.990703695919365e-06, 6.6349402914056554e-06, 6.355295226967428e-06, 6.129083431005711e-06, 5.99266377321328e-06, 5.8016053117171396e-06, 5.728539235860808e-06, 5.580269771598978e-06, 5.463929483084939e-06, 5.414202405518154e-06, 5.320095169736305e-06, 5.224529104452813e-06, 5.126427822688129e-06, 5.020667231292464e-06, 4.903477474726969e-06, 4.799095677299192e-06, 4.568513304548105e-06, 4.272334535926348e-06, 4.043248281959677e-06, 3.775350933210575e-06, 3.4793677059496986e-06, 3.160791720802081e-06, 2.923573902080534e-06, 2.8337476578599308e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.5575103993323864e-05, "first_to_last_ratio": 2.443213468991071, "imbalance_abs_log_ratio": 0.893314168315378, "last_quartile_mean": 1.456078416595119e-05, "mean": 2.3831100747884193e-05, "normalized": [ 1.614905261758675, 1.6568148874352646, 1.6283515751571318, 1.4605379751384047, 1.4717695441533916, 1.4559185813526447, 1.358376097224691, 1.295738430337987, 1.1776907075521434, 1.1141057290944703, 1.0640572841848701, 1.0269673379730528, 1.0041631647243947, 0.9733861101490422, 0.9597079874490758, 0.9349675920893245, 0.9158525986134215, 0.9066797587654788, 0.890357239209801, 0.8734858422464332, 0.8579185157197757, 0.8414715045701692, 0.8239471946120328, 0.8048352542714806, 0.7658980848440458, 0.7173626211575344, 0.6794957283273223, 0.6347225670052176, 0.5854538165358278, 0.5328910383364602, 0.49362355782826073, 0.47854641218217414 ], "peak_layer": 2, "peak_normalized": 1.6568148874352646, "population_cv": 0.33383138748240165, "rest_mean": 2.4630704299650257e-05, "spike_contrast": 0.7922323832463255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.951324156834744e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.848496999125928e-05, 3.94837225030642e-05, 3.880541044054553e-05, 3.48062276316341e-05, 3.507388828438707e-05, 3.4696142392931506e-05, 3.237159762647934e-05, 3.087887307628989e-05, 2.8065665901522152e-05, 2.6550365873845294e-05, 2.535765634092968e-05, 2.4473762096022256e-05, 2.393031354586128e-05, 2.3196862457552925e-05, 2.2870897737448104e-05, 2.2281306883087382e-05, 2.182577554776799e-05, 2.1607176677207462e-05, 2.1218193069216795e-05, 2.081612910842523e-05, 2.0445142581593245e-05, 2.0053192201885395e-05, 1.9635568605735898e-05, 1.9180110029992647e-05, 1.825219442253001e-05, 1.7095540897571482e-05, 1.6193131159525365e-05, 1.5126137441257015e-05, 1.3952008885098621e-05, 1.2699380022240803e-05, 1.176359273813432e-05, 1.1404287761251908e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03154835826717317, "first_to_last_ratio": 2.4432135220636924, "imbalance_abs_log_ratio": 0.8933141900378437, "last_quartile_mean": 0.012912648846395314, "mean": 0.021133658214239404, "normalized": [ 1.6149054410358779, 1.6568148414506838, 1.628351534517943, 1.460537924411657, 1.4717696782723049, 1.4559186942111129, 1.3583760820142625, 1.2957384038760298, 1.1776906753918865, 1.1141057069236509, 1.0640572599127858, 1.0269673356633326, 1.0041631806471485, 0.9733861162020112, 0.9597079595045668, 0.934967535297851, 0.9158525948921301, 0.9066797961300305, 0.890357194041685, 0.8734858545515346, 0.8579184935162031, 0.8414714427912773, 0.8239471886412217, 0.8048352448740587, 0.7658981573629571, 0.7173626162098946, 0.679495681209669, 0.6347225519706674, 0.5854538479398482, 0.5328910449060651, 0.49362353392574315, 0.4785463877039095 ], "peak_layer": 2, "peak_normalized": 1.6568148414506838, "population_cv": 0.33383140220751223, "rest_mean": 0.021842754652930632, "spike_contrast": 0.7922323772924418, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01730453744530678, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.034128859639167786, 0.035014558583498, 0.03441302478313446, 0.030866509303450584, 0.03110387735068798, 0.03076888807117939, 0.028707455843687057, 0.02738369256258011, 0.024888912215828896, 0.023545129224658012, 0.022487422451376915, 0.021703576669096947, 0.021221641451120377, 0.020571209490299225, 0.020282140001654625, 0.0197592843323946, 0.01935531571507454, 0.019161460921168327, 0.018816504627466202, 0.018459951505064964, 0.0181309562176466, 0.01778336986899376, 0.01741301827132702, 0.017009112983942032, 0.01618622988462448, 0.015160496346652508, 0.01436022948473692, 0.013414009474217892, 0.01237278152257204, 0.011261937208473682, 0.010432071052491665, 0.010113435797393322 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.3449252946884371e-06, "first_to_last_ratio": 1.4244745637229155, "imbalance_abs_log_ratio": 0.35380301851203094, "last_quartile_mean": 9.441553601163832e-07, "mean": 1.1411003768557748e-06, "normalized": [ 2.099491828546412, 1.269444086454149, 1.1995747754012265, 1.0098229278397233, 0.9820952443149423, 0.9671699039875943, 0.9480579483618607, 0.9533142816284239, 0.9297807852159039, 0.9211908612615112, 0.919724320468778, 0.9287522140783918, 0.9295613022426437, 0.9490597192634187, 0.9764897109479356, 0.9869635221203849, 0.989406229128281, 1.0074504593684357, 1.0338404225654974, 1.058785663749283, 1.0614877053824159, 1.080292006254891, 1.0793236111118636, 1.0996580161619744, 1.0501518033485862, 0.9775929043037735, 0.9270853191053471, 0.8657999542590211, 0.7904345026237674, 0.7106884577741329, 0.6594512386425827, 0.6380582740868475 ], "peak_layer": 1, "peak_normalized": 2.099491828546412, "population_cv": 0.2365628847316153, "rest_mean": 1.1254244832849104e-06, "spike_contrast": 1.0891447807859125, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2257502021384426e-06, "top_five_layers": [ 1, 2, 3, 24, 22 ], "top_five_spike_overlap": 2 }, "values": [ 2.3957309167599306e-06, 1.448563125450164e-06, 1.368835228277021e-06, 1.1523093235155102e-06, 1.120669253396045e-06, 1.1036379419238074e-06, 1.081829282156832e-06, 1.0878272860281868e-06, 1.0609732044031261e-06, 1.0511712389416061e-06, 1.049497768690344e-06, 1.0597995014904882e-06, 1.0607227522996254e-06, 1.082972403310123e-06, 1.114272777158476e-06, 1.126224447034474e-06, 1.1290118209217326e-06, 1.1496020988488453e-06, 1.1797156957982224e-06, 1.2081807199137984e-06, 1.2112640206396463e-06, 1.2327216154517373e-06, 1.2316165793890832e-06, 1.2548201766549028e-06, 1.1983286185568431e-06, 1.1155316315125674e-06, 1.0578974070085678e-06, 9.879646540866815e-07, 9.019651088237879e-07, 8.109668669931125e-07, 7.525000569330587e-07, 7.280885370164469e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6303443064534804e-05, "first_to_last_ratio": 2.692545967098127, "imbalance_abs_log_ratio": 0.9904872021768188, "last_quartile_mean": 9.768985705704836e-06, "mean": 1.6957480056589702e-05, "normalized": [ 1.6937361552633041, 1.7160304536137982, 1.6834660265718648, 1.5207187641970439, 1.5301774129520833, 1.5109128899887163, 1.4130194473875266, 1.3410670821715687, 1.2180739698019514, 1.1398798060375233, 1.078696685989418, 1.0334569745212194, 1.008882916344115, 0.9710505739450633, 0.9566696262711185, 0.9261570152534361, 0.9027531401068917, 0.895031047688981, 0.8707202243782249, 0.8467588255080271, 0.8234129285380357, 0.7960884736247433, 0.7684181877378946, 0.7461244793595948, 0.7082594203206751, 0.6648334442793923, 0.6336722392847842, 0.5977542493887017, 0.5540428877068647, 0.5092666262142399, 0.47765489699849767, 0.4632131285546998 ], "peak_layer": 2, "peak_normalized": 1.7160304536137982, "population_cv": 0.370371578681264, "rest_mean": 1.7684576926479133e-05, "spike_contrast": 0.7368656323168925, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3031156959186773e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.87214970740024e-05, 2.9099552193656564e-05, 2.854734157153871e-05, 2.578755811555311e-05, 2.5947952963178977e-05, 2.5621275199227966e-05, 2.3961249098647386e-05, 2.2741118300473318e-05, 2.065546505036764e-05, 1.932948907779064e-05, 1.8291977539774962e-05, 1.752482603478711e-05, 1.7108111933339387e-05, 1.6466570741613396e-05, 1.6222706108237617e-05, 1.5705289115430787e-05, 1.5308418369386345e-05, 1.5177471141214482e-05, 1.476522083976306e-05, 1.4358895896293689e-05, 1.3963008314021863e-05, 1.3499654414772522e-05, 1.3030436093686149e-05, 1.2652390978473704e-05, 1.2010294994979631e-05, 1.1273899872321635e-05, 1.0745484360086266e-05, 1.0136405762750655e-05, 9.395171218784526e-06, 8.635878657514695e-06, 8.099823389784433e-06, 7.854927389416844e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9079276171396486e-05, "first_to_last_ratio": 2.6285962135440917, "imbalance_abs_log_ratio": 0.9664499445808603, "last_quartile_mean": 7.258351843120181e-06, "mean": 1.2128285078460976e-05, "normalized": [ 1.7356747962648031, 1.7341306113468407, 1.7105708428728739, 1.5429534394890942, 1.5518170549190342, 1.5419721261714732, 1.4249221294985488, 1.3429376987128008, 1.218113684845076, 1.1171416737474187, 1.0512005083729412, 1.010591189648245, 0.9869796783812811, 0.9490114917860836, 0.945008399357438, 0.9108007889697958, 0.8807169292049036, 0.8929396267645643, 0.8504587187244783, 0.8212221659423008, 0.7893862502127975, 0.7583794939936823, 0.7281575784579098, 0.7171946304338384, 0.6937537204046544, 0.6568753570132616, 0.6340749327739144, 0.6088859412265378, 0.578600809495843, 0.5544537228265827, 0.5337941004053197, 0.5272799077356636 ], "peak_layer": 1, "peak_normalized": 1.7356747962648031, "population_cv": 0.37524903134285426, "rest_mean": 1.27181367335854e-05, "spike_contrast": 0.7031758132598661, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.943086140789091e-06, "top_five_layers": [ 1, 2, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.1050758732599206e-05, 2.1032030417700298e-05, 2.074629082926549e-05, 1.8713379176915623e-05, 1.882087963167578e-05, 1.8701477529248223e-05, 1.7281861801166087e-05, 1.6287531252601184e-05, 1.4773630027775653e-05, 1.3549012692237739e-05, 1.2749259440170135e-05, 1.2256738045834936e-05, 1.1970370906055905e-05, 1.1509881915117148e-05, 1.1461331268947106e-05, 1.1046451618312858e-05, 1.0681585990823805e-05, 1.0829826351255178e-05, 1.031460578815313e-05, 9.96001654129941e-06, 9.573901479598135e-06, 9.197842700814363e-06, 8.831302693579346e-06, 8.698340934643056e-06, 8.414042895310558e-06, 7.966771590872668e-06, 7.690241545788012e-06, 7.384742275462486e-06, 7.017435564193875e-06, 6.724572813254781e-06, 6.474007022916339e-06, 6.395001037162729e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.327860384975793e-05, "first_to_last_ratio": 2.380013015374498, "imbalance_abs_log_ratio": 0.8671059563131774, "last_quartile_mean": 3.078916097365436e-05, "mean": 4.963039305039274e-05, "normalized": [ 1.603330252581107, 1.6263391318078697, 1.606460956677952, 1.4448408316016137, 1.4626423772053405, 1.419568110995152, 1.3492771702415232, 1.299432930512805, 1.1824076365210394, 1.115525432190159, 1.069721713913707, 1.0194221990001253, 0.9991025293542795, 0.9701679624855374, 0.9607540796701428, 0.941138763470014, 0.9221596303852333, 0.9137187507583548, 0.8936702231039447, 0.8650472606162771, 0.8595412236639356, 0.8548091765162906, 0.8394520870170261, 0.8185169785606647, 0.7782339575226548, 0.7312396774119493, 0.6924473875661084, 0.6467258401932515, 0.5884903459061062, 0.5417511914242714, 0.5024863639066834, 0.48157782721888 ], "peak_layer": 2, "peak_normalized": 1.6263391318078697, "population_cv": 0.3240577046299135, "rest_mean": 5.1191814068092584e-05, "spike_contrast": 0.8047911625091714, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.119871955481358e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.957391062518582e-05, 8.071585034485906e-05, 7.97292887000367e-05, 7.17080183676444e-05, 7.259151607286185e-05, 7.045372331049293e-05, 6.696515629300848e-05, 6.449136708397418e-05, 5.8683355746325105e-05, 5.5363965657306835e-05, 5.309070911607705e-05, 5.059432442067191e-05, 4.958585122949444e-05, 4.81498173030559e-05, 4.768260259879753e-05, 4.67090867459774e-05, 4.576714491122402e-05, 4.534822073765099e-05, 4.435320443008095e-05, 4.293263555155136e-05, 4.265936877345666e-05, 4.242451541358605e-05, 4.1662337025627494e-05, 4.062331936438568e-05, 3.862405719701201e-05, 3.629171260399744e-05, 3.436643601162359e-05, 3.2097257644636557e-05, 2.920700717368163e-05, 2.6887324565905146e-05, 2.493859574315138e-05, 2.3900896849227138e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.062413111329078674, 0.07552354782819748, 0.14653873443603516, 0.2105521410703659, 0.2186809629201889, 0.14272794127464294, 0.1287408471107483, 0.13253600895404816, 0.13371841609477997, 0.09585829079151154, 0.09883332997560501, 0.10020580142736435, 0.11237040907144547, 0.08538161963224411, 0.08663177490234375, 0.10358677804470062, 0.10874549299478531, 0.06467291712760925, 0.07278503477573395, 0.08346924930810928, 0.09857872873544693, 0.06093711405992508, 0.06834813207387924, 0.0680762529373169, 0.07369214296340942, 0.05633483827114105, 0.05850831791758537, 0.06479237228631973, 0.06705968081951141, 0.061747051775455475, 0.0654391348361969, 0.07205834239721298 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.896821650390848e-06, "first_to_last_ratio": 3.086921357209413, "imbalance_abs_log_ratio": 1.1271742697207559, "last_quartile_mean": 1.5863124076531676e-06, "mean": 3.5260151562965802e-06, "normalized": [ 2.4218303084255877, 1.2262927966730428, 0.9438390302215882, 1.3886420618812072, 0.4539768042487178, 1.1428851913549358, 1.7589416507010018, 1.7737466699170399, 1.825038365263652, 1.4555799135606915, 1.1601907943732277, 1.2524020951634824, 0.6438615738360847, 1.151558369997547, 1.2084021893674217, 1.345408620035675, 0.6220571382439607, 0.9702308357427584, 1.0337141829385006, 1.2671021334514148, 0.7373548929504563, 0.9774444054740276, 0.9713309430114792, 0.6690638964335255, 0.5006396015841585, 0.6207090232052809, 0.3879420916285575, 0.589315985595003, 0.2987117204227787, 0.45844781171783394, 0.36830553154718904, 0.37503337103217216 ], "peak_layer": 1, "peak_normalized": 2.4218303084255877, "population_cv": 0.49741945720200614, "rest_mean": 3.6754354368895292e-06, "spike_contrast": 0.7398159178102259, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7191456410946557e-06, "top_five_layers": [ 1, 9, 8, 7, 10 ], "top_five_spike_overlap": 0 }, "values": [ 8.539410373487044e-06, 4.32392698712647e-06, 3.327990725665586e-06, 4.89637295686407e-06, 1.6007290923880646e-06, 4.029830506624421e-06, 6.202054919413058e-06, 6.25425764155807e-06, 6.435112936742371e-06, 5.1323968364158645e-06, 4.09085032515577e-06, 4.415988769324031e-06, 2.2702656679030042e-06, 4.060412265971536e-06, 4.260844434611499e-06, 4.743931185657857e-06, 2.1933828975306824e-06, 3.421048631935264e-06, 3.644891876319889e-06, 4.467821327125421e-06, 2.5999245281127514e-06, 3.446483788138721e-06, 3.4249276268383255e-06, 2.3591294393554563e-06, 1.7652628230280243e-06, 2.188629423471866e-06, 1.3678896948476904e-06, 2.0779370970558375e-06, 1.0532620535741444e-06, 1.6164939324880834e-06, 1.2986508863832569e-06, 1.3223733503764379e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.0633759504562477e-05, "first_to_last_ratio": 3.384548225953592, "imbalance_abs_log_ratio": 1.2192204339279695, "last_quartile_mean": 6.096459003401833e-06, "mean": 1.4159107664113435e-05, "normalized": [ 3.1246660167166214, 1.2194113419753856, 0.9163650527941182, 1.3371893358177132, 0.4570845763492036, 1.1361310632504436, 1.7660054721762457, 1.7013730653458237, 1.7160657990991948, 1.4561034988782278, 1.164959879782242, 1.19728584675021, 0.6527944132565722, 1.141018877615477, 1.2123794000378127, 1.3469149905738007, 0.5843570471425387, 0.938924512245779, 1.003661170695441, 1.2206463029313366, 0.695795797421837, 0.9665811243416496, 0.9496890218619827, 0.6500522038437716, 0.48232230036649704, 0.5819961943923555, 0.38373225704750635, 0.5379015492539144, 0.28024614530736863, 0.446258402189272, 0.3666855435712217, 0.36540179696843594 ], "peak_layer": 1, "peak_normalized": 3.1246660167166214, "population_cv": 0.5666838267588304, "rest_mean": 1.4817537400328243e-05, "spike_contrast": 0.7156106174780821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0603587088553468e-05, "top_five_layers": [ 1, 7, 9, 8, 10 ], "top_five_spike_overlap": 0 }, "values": [ 4.4242482545087114e-05, 1.726577647787053e-05, 1.2974911442142911e-05, 1.8933407773147337e-05, 6.471909728134051e-06, 1.60866020451067e-05, 2.5005061615956947e-05, 2.408992440905422e-05, 2.4297960408148356e-05, 2.0617126210709102e-05, 1.649479236220941e-05, 1.6952499208855443e-05, 9.242986379831564e-06, 1.615580913494341e-05, 1.7166210454888642e-05, 1.9071114365942776e-05, 8.273974344774615e-06, 1.3294333257363178e-05, 1.4210946574166883e-05, 1.7283262423006818e-05, 9.85184760793345e-06, 1.3685926205653232e-05, 1.344674910797039e-05, 9.204159141518176e-06, 6.829253379692091e-06, 8.240546776505653e-06, 5.433306341728894e-06, 7.616205948579591e-06, 3.96803534385981e-06, 6.318620762613136e-06, 5.1919400902988855e-06, 5.173763383936603e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.01829822437139228, "first_to_last_ratio": 3.3845482144866907, "imbalance_abs_log_ratio": 1.219220430539954, "last_quartile_mean": 0.005406400858191773, "mean": 0.012556438210594933, "normalized": [ 3.1246659927740246, 1.2194113871462446, 0.9163651029165816, 1.337189393706543, 0.45708456224334865, 1.136131091380754, 1.7660054527899205, 1.701373061401254, 1.7160657272955568, 1.4561034762492286, 1.1649599187562338, 1.1972858308410934, 0.6527944366098802, 1.1410188066701286, 1.2123793905582598, 1.3469149814270192, 0.5843570399372514, 0.9389244849330981, 1.0036612348062732, 1.2206462587941367, 0.6957957692173607, 0.9665811150881268, 0.9496889847306975, 0.6500522635247569, 0.48232233035374666, 0.5819961795520987, 0.38373222955368735, 0.5379015675830991, 0.28024616194939816, 0.44625838932572864, 0.3666855640843364, 0.3654018138001321 ], "peak_layer": 1, "peak_normalized": 3.1246659927740246, "population_cv": 0.566683818054122, "rest_mean": 0.013140340275303633, "spike_contrast": 0.715610620741758, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.009403367061167955, "top_five_layers": [ 1, 7, 9, 8, 10 ], "top_five_spike_overlap": 0 }, "values": [ 0.03923467546701431, 0.015311463735997677, 0.011506281793117523, 0.01679033599793911, 0.005739354062825441, 0.014265759848058224, 0.022174738347530365, 0.021363185718655586, 0.021547673270106316, 0.018283473327755928, 0.014627747237682343, 0.015033645555377007, 0.008196773007512093, 0.014327132143080235, 0.015223166905343533, 0.01691245473921299, 0.0073374430648982525, 0.01178954727947712, 0.012602410279214382, 0.01532696932554245, 0.008736716583371162, 0.012136816047132015, 0.011924711056053638, 0.008162341080605984, 0.006056250538676977, 0.007307799067348242, 0.004818310029804707, 0.00675412779673934, 0.003518893616273999, 0.00560341589152813, 0.0046042646281421185, 0.004588145297020674 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.353687545308958e-06, "first_to_last_ratio": 2.0714788069877605, "imbalance_abs_log_ratio": 0.7282627517264126, "last_quartile_mean": 1.1362353973254358e-06, "mean": 1.408354910914511e-06, "normalized": [ 8.562446711412006, 2.3789661370966213, 0.4619949054197906, 0.5268622892149433, 0.20998667107627778, 0.31572440518666567, 0.4964571094095077, 0.41741635581071684, 0.503305459225638, 0.6375462734944312, 0.6385460296455763, 0.6762080183809729, 0.5854533273996188, 0.7765606105827553, 0.8348084946779258, 0.7212436224371831, 0.4309937062730567, 0.8125935658899578, 0.874841120374869, 0.8623676992823301, 0.6400613645240014, 1.1225318571258882, 0.9563878029262699, 1.1024404313909701, 0.7907538382800698, 1.0058032833852923, 0.7964771092624024, 1.1595820855067065, 0.6910708445662769, 0.7332770216661861, 0.6924978683240438, 0.5847939807510477 ], "peak_layer": 1, "peak_normalized": 8.562446711412006, "population_cv": 1.4067449803535663, "rest_mean": 1.4285843490226938e-06, "spike_contrast": 0.9093729369351278, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2991159451303246e-06, "top_five_layers": [ 1, 2, 28, 22, 24 ], "top_five_spike_overlap": 2 }, "values": [ 1.2058963875460904e-05, 3.3504286420793505e-06, 6.506527938654472e-07, 7.420090923915268e-07, 2.957357594368659e-07, 4.446520165402035e-07, 6.991878080953029e-07, 5.878703746020619e-07, 7.088327151905105e-07, 8.978914252111281e-07, 8.992994366963103e-07, 9.523408834866132e-07, 8.245260687544942e-07, 1.0936729495369946e-06, 1.1757066431528074e-06, 1.0157669976251782e-06, 6.069921028029057e-07, 1.1444201390986564e-06, 1.2320867881498998e-06, 1.214519784298318e-06, 9.014335660140205e-07, 1.580923253641231e-06, 1.3469334589899518e-06, 1.5526273955401848e-06, 1.1136620514662354e-06, 1.4165279935696162e-06, 1.121722448260698e-06, 1.6331031247318606e-06, 9.732730177347548e-07, 1.0327142945243395e-06, 9.752827736519976e-07, 8.23597474663984e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2558051935229742e-05, "first_to_last_ratio": 3.235991308724262, "imbalance_abs_log_ratio": 1.174335313384314, "last_quartile_mean": 3.880743406625697e-06, "mean": 9.272452984987467e-06, "normalized": [ 1.947517132469556, 1.2659703720728832, 1.00333727887001, 1.3279268679179317, 0.5164022830958469, 1.2531052602985686, 1.8259856606852272, 1.694473384142606, 1.5673596610865692, 1.56833061106792, 1.2405222428920049, 1.2135192597759437, 0.7282554475336783, 1.2012094109417673, 1.3067699865079492, 1.2637529494818742, 0.6470120244072932, 1.0867341236920782, 1.1179134962672703, 1.2614932519699678, 0.7458582439755571, 1.0936916340494032, 1.0525562723887423, 0.7221117539250419, 0.5393185760918967, 0.6195458807889652, 0.4064975180219839, 0.4502100069225183, 0.3003691735935886, 0.42059397269238735, 0.3160004972452703, 0.29565576512769975 ], "peak_layer": 1, "peak_normalized": 1.947517132469556, "population_cv": 0.45871035705065244, "rest_mean": 9.563149029013908e-06, "spike_contrast": 0.8054558518198625, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.702694347244687e-06, "top_five_layers": [ 1, 7, 8, 10, 9 ], "top_five_spike_overlap": 0 }, "values": [ 1.8058261048281565e-05, 1.17386507554329e-05, 9.303397746407427e-06, 1.2313139450270683e-05, 4.7883158913464285e-06, 1.161935961135896e-05, 1.6931366189965047e-05, 1.5711924788774922e-05, 1.4533268767991103e-05, 1.4542271856043953e-05, 1.1502684174047317e-05, 1.125230028264923e-05, 6.752714398317039e-06, 1.1138157788082026e-05, 1.2116963262087665e-05, 1.171808980870992e-05, 5.999388577038189e-06, 1.007669106911635e-05, 1.0365800335421227e-05, 1.1697136869770475e-05, 6.915935500728665e-06, 1.014120425679721e-05, 9.759778549778275e-06, 6.69574728817679e-06, 5.0008061407424975e-06, 5.7447100516583305e-06, 3.7692291243729414e-06, 4.174551122559933e-06, 2.7851590402860893e-06, 3.899937837559264e-06, 2.9300997539394302e-06, 2.741454181887093e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 7.895624435150239e-06, "first_to_last_ratio": 3.058028124449071, "imbalance_abs_log_ratio": 1.1177703044721228, "last_quartile_mean": 2.581933230771938e-06, "mean": 6.122625247684255e-06, "normalized": [ 1.1319461307239276, 1.3085216761923217, 1.0639933002407227, 1.2290715675273767, 0.5847062304385443, 1.3366939702161282, 1.897298452770328, 1.7644209127072272, 1.3831256030397758, 1.6077197781270347, 1.29630788384933, 1.1373671133182328, 0.7956691079721259, 1.243813918811557, 1.3698450981038646, 1.232490514336823, 0.692747757146363, 1.2511518200321439, 1.1980660456404937, 1.2681707953518462, 0.7704716066493089, 1.1938208830614028, 1.1123242489274907, 0.7566266249200412, 0.584885451820435, 0.6871683496836065, 0.42379305915423937, 0.4006394094629471, 0.32523481684807415, 0.389650977401113, 0.2995172170061151, 0.2627396785190583 ], "peak_layer": 7, "peak_normalized": 1.897298452770328, "population_cv": 0.4334998696783958, "rest_mean": 6.25457262567916e-06, "spike_contrast": 0.8649846648673781, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.410109406511765e-06, "top_five_layers": [ 7, 8, 10, 9, 15 ], "top_five_spike_overlap": 0 }, "values": [ 6.930481958988821e-06, 8.01158785179723e-06, 6.5144322434207425e-06, 7.52514461055398e-06, 3.579937128961319e-06, 8.184076250472572e-06, 1.1616447409323882e-05, 1.0802888027683366e-05, 8.468359737889841e-06, 9.843465704761911e-06, 7.936807378428057e-06, 6.9636726038879715e-06, 4.871583769272547e-06, 7.615406502736732e-06, 8.387048183067236e-06, 7.546077540609986e-06, 4.241434908180963e-06, 7.660333722014911e-06, 7.335309419431724e-06, 7.764534529997036e-06, 4.717308911494911e-06, 7.309317879844457e-06, 6.810344530094881e-06, 4.6325412768055685e-06, 3.581034434319008e-06, 4.207274287182372e-06, 2.594726083771093e-06, 2.45296496359515e-06, 1.9912909010599833e-06, 2.3856869120209012e-06, 1.8338316749577643e-06, 1.6086565892692306e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.8174724750206224e-05, "first_to_last_ratio": 3.4131964974538564, "imbalance_abs_log_ratio": 1.2276492415031157, "last_quartile_mean": 1.1184449761003634e-05, "mean": 2.727851558859129e-05, "normalized": [ 2.560639234860951, 1.2774901614756382, 0.9638213100637651, 1.437728242920233, 0.4488834649809974, 1.1881397139629066, 1.757440362013926, 1.5614006473172555, 1.6987735612467263, 1.5843521015029483, 1.2353981365841489, 1.257208792964521, 0.6596706652646716, 1.1343619132828286, 1.279586514141272, 1.2620503153781926, 0.6179136513723597, 0.948834235599364, 1.071059237777408, 1.2838045586802798, 0.7435873864059896, 1.0227779780310893, 1.0169155522739846, 0.7080864895405166, 0.5037032405562084, 0.5772180835558656, 0.3939239236704607, 0.465560531237428, 0.2747838533410833, 0.43521029512005854, 0.3246502639074058, 0.3050255809695173 ], "peak_layer": 1, "peak_normalized": 2.560639234860951, "population_cv": 0.5127605859467733, "rest_mean": 2.829381118216487e-05, "spike_contrast": 0.7703422929828946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.179591938329395e-05, "top_five_layers": [ 1, 7, 9, 10, 8 ], "top_five_spike_overlap": 0 }, "values": [ 6.985043728491291e-05, 3.48480352840852e-05, 2.6291614631190896e-05, 3.921909228665754e-05, 1.224487459694501e-05, 3.241068770876154e-05, 4.79403643112164e-05, 4.2592691897880286e-05, 4.634002107195556e-05, 4.321877349866554e-05, 3.3699827326927334e-05, 3.429478965699673e-05, 1.799483652575873e-05, 3.094370913458988e-05, 3.4905220672953874e-05, 3.442685920163058e-05, 1.6855767171364278e-05, 2.588278948678635e-05, 2.9216906114015728e-05, 3.502028266666457e-05, 2.0283960111555643e-05, 2.7899865017388947e-05, 2.773994674498681e-05, 1.9315548343001865e-05, 1.374027669953648e-05, 1.574565249029547e-05, 1.0745659892563708e-05, 1.2699800208793022e-05, 7.495695626857923e-06, 1.1871890819747932e-05, 8.855977284838445e-06, 8.320645065396093e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05931045487523079, 0.07934419065713882, 0.214645653963089, 0.09514089673757553, 0.12821291387081146, 0.1321839988231659, 0.12568901479244232, 0.1335759311914444, 0.07788944244384766, 0.08352623134851456, 0.07934392243623734, 0.09343641996383667, 0.07063088566064835, 0.06870647519826889, 0.07622843235731125, 0.08666495233774185, 0.057347409427165985, 0.05777324363589287, 0.06351501494646072, 0.07469023764133453, 0.051319729536771774, 0.05102822557091713, 0.05902284011244774, 0.06656526774168015, 0.050236333161592484, 0.05184732377529144, 0.061248794198036194, 0.06608261168003082, 0.048912134021520615, 0.062179163098335266, 0.07131517678499222, 0.06802479177713394 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.536313694179171e-06, "first_to_last_ratio": 0.32103533438342263, "imbalance_abs_log_ratio": 1.136204085949217, "last_quartile_mean": 7.90041912068773e-06, "mean": 5.024183989732478e-06, "normalized": [ 1.4240775191442525, 0.5380590571289816, 0.1905335652531641, 0.6822754406556624, 0.2910317874157833, 0.1585066509591937, 0.2416971045584352, 0.5123870473279437, 0.7644347964709719, 0.6121884879234236, 0.6815584070993507, 0.7497887841806089, 0.7911538454261898, 0.7760261752123336, 0.7322264401935603, 0.7150908135381492, 0.8515627911494139, 0.9152360407363572, 1.0880793237379123, 1.1939585333655534, 1.1057515489086724, 1.3427709644104338, 1.488668647713089, 1.573111703813592, 1.6787238134633642, 1.809834874575413, 1.572907328432356, 1.5205734730600422, 1.9211083090017804, 1.5699327524673685, 1.322832236174336, 1.1839117365023109 ], "peak_layer": 29, "peak_normalized": 1.9211083090017804, "population_cv": 0.48692469008524586, "rest_mean": 4.616847960463659e-06, "spike_contrast": 1.5646602638087805, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.223798547784099e-06, "top_five_layers": [ 29, 26, 25, 24, 27 ], "top_five_spike_overlap": 2 }, "values": [ 7.1548274718225e-06, 2.703307700357982e-06, 9.572756880515954e-07, 3.4278773455298506e-06, 1.4621972468376043e-06, 7.963665780152951e-07, 1.214330723087187e-06, 2.5743267997313524e-06, 3.840661065623863e-06, 3.0757475997233996e-06, 3.424274837016128e-06, 3.767076805161196e-06, 3.974902483605547e-06, 3.898898285115138e-06, 3.6788403576792916e-06, 3.5927478165831417e-06, 4.278408141544787e-06, 4.598314262693748e-06, 5.466710717882961e-06, 5.9986673477396835e-06, 5.555499228648841e-06, 6.74632838126854e-06, 7.4793451858568005e-06, 7.903602636361029e-06, 8.434217306785285e-06, 9.092943400901277e-06, 7.902575816842727e-06, 7.639640898560174e-06, 9.65200160862878e-06, 7.887630999903195e-06, 6.646152542089112e-06, 5.948190391791286e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.0282310029197106e-05, "first_to_last_ratio": 0.32628005375076213, "imbalance_abs_log_ratio": 1.1199992058116255, "last_quartile_mean": 3.151375608467788e-05, "mean": 2.007631842104729e-05, "normalized": [ 1.459931390393604, 0.545480683473369, 0.19734981318978928, 0.6879078045288014, 0.29465579930095187, 0.15846117316623343, 0.24241735602570375, 0.5110850599423152, 0.7578637625848365, 0.6102749921347288, 0.6808791746019699, 0.7378534750263886, 0.7874390409852465, 0.7700487415879345, 0.7271376355617218, 0.7086964687616041, 0.8475748227839637, 0.9199112127298047, 1.0869552322670417, 1.1894614876142768, 1.1112082208533107, 1.3353228072817924, 1.4957799353947956, 1.5787202238925846, 1.6775886487367329, 1.8106175804235458, 1.5746959684661916, 1.5206109758006878, 1.9184410970777135, 1.5638430908546315, 1.3084920518741907, 1.1832942726835374 ], "peak_layer": 29, "peak_normalized": 1.9184410970777135, "population_cv": 0.4874445604692638, "rest_mean": 1.8441496498174576e-05, "spike_contrast": 1.5673541899064982, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8904356804559937e-05, "top_five_layers": [ 29, 26, 25, 24, 27 ], "top_five_spike_overlap": 2 }, "values": [ 2.9310047466424294e-05, 1.0951243893941864e-05, 3.962057689932408e-06, 1.3810656128043775e-05, 5.915603651374113e-06, 3.181316969858017e-06, 4.866848030360416e-06, 1.0260706403641962e-05, 1.5215114217426162e-05, 1.2252075066498946e-05, 1.3669547115569003e-05, 1.4813381312706042e-05, 1.5808876923983917e-05, 1.5459743735846132e-05, 1.4598246707464568e-05, 1.4228015970729757e-05, 1.7016182027873583e-05, 1.846843042585533e-05, 2.1822059352416545e-05, 2.3880007574916817e-05, 2.2308970073936507e-05, 2.6808365873876028e-05, 3.0029754270799458e-05, 3.16948899126146e-05, 3.36798038915731e-05, 3.6350535083329305e-05, 3.1614097679266706e-05, 3.052827014471404e-05, 3.851523433695547e-05, 3.139621185255237e-05, 2.6269703084835783e-05, 2.3756192604196258e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.009118455491261557, "first_to_last_ratio": 0.3262800607417022, "imbalance_abs_log_ratio": 1.1199991843854307, "last_quartile_mean": 0.027946713846176863, "mean": 0.017803879898565356, "normalized": [ 1.4599314269226145, 0.5454806821468461, 0.19734980888248807, 0.6879078250232088, 0.29465581055680906, 0.15846116652987666, 0.2424173583964748, 0.5110850711933042, 0.7578637834127073, 0.6102749946338123, 0.6808791796688727, 0.7378534997474308, 0.7874390718063808, 0.7700486867302203, 0.7271376730207798, 0.7086964305299408, 0.847574856844415, 0.919911238227211, 1.0869552482975884, 1.1894614745664287, 1.1112081941897007, 1.3353228272158537, 1.4957798460287703, 1.5787202151640234, 1.6775885884762012, 1.810617543503823, 1.5746959998764927, 1.5206111322850961, 1.918440985885323, 1.5638431207118677, 1.3084920400476525, 1.1832942194777856 ], "peak_layer": 29, "peak_normalized": 1.918440985885323, "population_cv": 0.4874445523784538, "rest_mean": 0.01635410358129, "spike_contrast": 1.5673541435299125, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.025632672011852264, "top_five_layers": [ 29, 26, 25, 24, 27 ], "top_five_spike_overlap": 2 }, "values": [ 0.025992443785071373, 0.00971167255192995, 0.0035135922953486443, 0.012247428297996521, 0.005246016662567854, 0.0028212235774844885, 0.004315969534218311, 0.009099297225475311, 0.013492915779352188, 0.01086526270955801, 0.012122291140258312, 0.01313665509223938, 0.014019470661878586, 0.01370985433459282, 0.01294587180018425, 0.012617546133697033, 0.01509012095630169, 0.016377989202737808, 0.01935202069580555, 0.021177029237151146, 0.01978381723165512, 0.023773927241563797, 0.026630684733390808, 0.02810734510421753, 0.029867585748434067, 0.032236017286777496, 0.02803569845855236, 0.027072777971625328, 0.03415569290518761, 0.027842475101351738, 0.023296235129237175, 0.02106722816824913 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.048277736425689e-07, "first_to_last_ratio": 0.2857864675257622, "imbalance_abs_log_ratio": 1.2525103640623867, "last_quartile_mean": 2.116362537663008e-06, "mean": 1.2933788453572959e-06, "normalized": [ 1.5722659861144699, 0.5494596534444791, 0.14380470914509727, 0.6710949567625404, 0.22255230509223908, 0.12256240809816367, 0.18626377655013246, 0.27306728115170176, 0.6044679656212456, 0.5171048294185702, 0.6237768050810616, 0.6102501003317593, 0.7436430071434332, 0.7420143247015445, 0.7014560094306781, 0.6999767554604397, 0.8688625721017827, 0.9411031514274796, 1.1366340862434177, 1.2603718481576007, 1.1663554971537906, 1.3398176046889787, 1.583025715064311, 1.629627005627822, 1.7375189329126879, 1.8474466033760384, 1.712075834943869, 1.6006522975801964, 1.9409014549228467, 1.6098356440532264, 1.3816138029988174, 1.2603970751995792 ], "peak_layer": 29, "peak_normalized": 1.9409014549228467, "population_cv": 0.5453879627305627, "rest_mean": 1.1757127585572864e-06, "spike_contrast": 1.6405161039879685, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9287757140773467e-06, "top_five_layers": [ 29, 26, 25, 27, 24 ], "top_five_spike_overlap": 2 }, "values": [ 2.033535565715283e-06, 7.106594921424403e-07, 1.859939686710277e-07, 8.679800203026389e-07, 2.878444433918048e-07, 1.5851962587021262e-07, 2.409096282462997e-07, 3.5317944480084407e-07, 7.818060794306803e-07, 6.688124472020718e-07, 8.067797239164065e-07, 7.892845701462647e-07, 9.61812133937201e-07, 9.597056305210572e-07, 9.072483635463868e-07, 9.053351277543698e-07, 1.123768470279174e-06, 1.2172029073553858e-06, 1.4700984820592566e-06, 1.6301382856909186e-06, 1.5085395261849044e-06, 1.732891746542009e-06, 2.047451971520786e-06, 2.10772509490198e-06, 2.247270231237053e-06, 2.3894483547337586e-06, 2.2143626665638294e-06, 2.0702498204627773e-06, 2.510320882720407e-06, 2.082127366520581e-06, 1.786950065252313e-06, 1.6301709138133447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.5630383971893025e-06, "first_to_last_ratio": 0.33029070877691014, "imbalance_abs_log_ratio": 1.1077820766301152, "last_quartile_mean": 1.9870490518769657e-05, "mean": 1.2924320181184612e-05, "normalized": [ 1.4791032175094332, 0.509681391761457, 0.16554536291443128, 0.6646319035237539, 0.2907163787243681, 0.15580076690764444, 0.2616909410389207, 0.5352725058862023, 0.8032176162539202, 0.6632295542966498, 0.7433292538374848, 0.8060642557026766, 0.8297942030089529, 0.8051034128510668, 0.768957453371839, 0.7423222477435537, 0.8644340163366496, 0.9439214195795605, 1.097648605420341, 1.189066040117497, 1.0986102223507093, 1.338134608877611, 1.4379233534932891, 1.5062047000334828, 1.6649733200863754, 1.7681836335559407, 1.5403186324010156, 1.4902361589548763, 1.9021040439797992, 1.5128593847489753, 1.2614380410420527, 1.1594833536894698 ], "peak_layer": 29, "peak_normalized": 1.9021040439797992, "population_cv": 0.47010582540802626, "rest_mean": 1.1945017537005629e-05, "spike_contrast": 1.5246988464712319, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8212554459751117e-05, "top_five_layers": [ 29, 26, 25, 27, 30 ], "top_five_spike_overlap": 1 }, "values": [ 1.911640356411226e-05, 6.587285497516859e-06, 2.139561274816515e-06, 8.589915523771197e-06, 3.75731156054826e-06, 2.0136189959885087e-06, 3.3821775105025154e-06, 6.918033250258304e-06, 1.0381041647633538e-05, 8.571791113354266e-06, 9.607025276636705e-06, 1.0417832527309656e-05, 1.072452596417861e-05, 1.0405414286651649e-05, 9.938252333085984e-06, 9.594010407454334e-06, 1.117222200264223e-05, 1.2199542652524542e-05, 1.4186362022883259e-05, 1.5367870219051838e-05, 1.4198790267982986e-05, 1.7294480130658485e-05, 1.8584181816549972e-05, 1.9466671801637858e-05, 2.151864828192629e-05, 2.285257141920738e-05, 1.9907571186195128e-05, 1.9260289263911545e-05, 2.4583401682320982e-05, 1.9552679077605717e-05, 1.6303229131153785e-05, 1.4985534107836429e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.125818321654151e-06, "first_to_last_ratio": 0.323071664836056, "imbalance_abs_log_ratio": 1.1298811078121074, "last_quartile_mean": 1.2770597891176294e-05, "mean": 8.254944056318436e-06, "normalized": [ 1.4528370129337378, 0.4887545271444068, 0.1464999865512414, 0.6321858434529055, 0.27861137744593806, 0.1604870466652074, 0.2845859883398777, 0.5544355837055989, 0.8055064100672278, 0.6948443934131001, 0.7505943725961226, 0.8501485246227231, 0.8625059475864164, 0.7928262823143416, 0.7999235841614649, 0.7664878864471988, 0.8659165482877662, 0.9625161170669155, 1.085645016298278, 1.1880281512105406, 1.0543357106739477, 1.3441904964711768, 1.3657148728576123, 1.4362250429780776, 1.6635267721014393, 1.753447817081851, 1.5686511233757214, 1.557344226245061, 1.8968012391308895, 1.5319357137581802, 1.2230350653804842, 1.1814513196345497 ], "peak_layer": 29, "peak_normalized": 1.8968012391308895, "population_cv": 0.4662704135356736, "rest_mean": 7.685049350011806e-06, "spike_contrast": 1.4746002210586762, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1332375470374245e-05, "top_five_layers": [ 29, 26, 25, 27, 28 ], "top_five_spike_overlap": 1 }, "values": [ 1.1993088264716789e-05, 4.034641278849449e-06, 1.2093491932319012e-06, 5.21865877090022e-06, 2.2999213342700386e-06, 1.3248115919850534e-06, 2.349241412957781e-06, 4.576834726321977e-06, 6.649410352110863e-06, 5.73590159547166e-06, 6.196114554768428e-06, 7.0179285103222355e-06, 7.119938345567789e-06, 6.544736606883816e-06, 6.603324436582625e-06, 6.327314622467384e-06, 7.148092663555872e-06, 7.945516699692234e-06, 8.961938874563202e-06, 9.807105925574433e-06, 8.703482308192179e-06, 1.1096217349404469e-05, 1.1273899872321635e-05, 1.1855957382067572e-05, 1.373232043988537e-05, 1.4474613635684364e-05, 1.2949127267347649e-05, 1.28557894640835e-05, 1.565798811498098e-05, 1.2646043614950031e-05, 1.0096086043631658e-05, 9.7528145488468e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.0554489708501933e-05, "first_to_last_ratio": 0.304193026420869, "imbalance_abs_log_ratio": 1.1900928237429556, "last_quartile_mean": 6.75705487083178e-05, "mean": 4.220494211892856e-05, "normalized": [ 1.4416703556828947, 0.48115035856614824, 0.16644163758868316, 0.6569569491237993, 0.2862080852971654, 0.14160291640607714, 0.23550628823607553, 0.48659285300774935, 0.769034088597553, 0.5943108767709541, 0.6762795951535582, 0.7435203621700468, 0.778579992579065, 0.7456281602454271, 0.7046027708376329, 0.7063643980108446, 0.8365760311947629, 0.8942705494380042, 1.096453061256065, 1.1557136235607102, 1.1214101214270462, 1.3312370970047724, 1.5301222792609177, 1.6116848329794238, 1.7202394076902243, 1.8048142344035467, 1.596568646119442, 1.523076115359806, 2.001887880882806, 1.5733919180733085, 1.3426993534506797, 1.2454051596248104 ], "peak_layer": 29, "peak_normalized": 2.001887880882806, "population_cv": 0.5063397050013957, "rest_mean": 3.858673785038668e-05, "spike_contrast": 1.6001157031841702, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.174324516905472e-05, "top_five_layers": [ 29, 26, 25, 24, 27 ], "top_five_spike_overlap": 2 }, "values": [ 6.0845613916171715e-05, 2.0306923033786006e-05, 7.024659680610057e-06, 2.772683001239784e-05, 1.2079395673936233e-05, 5.976342890789965e-06, 9.939529263647273e-06, 2.0536623196676373e-05, 3.24570391967427e-05, 2.5082856154767796e-05, 2.854234116966836e-05, 3.138023384963162e-05, 3.285992352175526e-05, 3.1469193345401436e-05, 2.9737719160038978e-05, 2.981206853291951e-05, 3.530764297465794e-05, 3.7742636777693406e-05, 4.627573798643425e-05, 4.877682658843696e-05, 4.732904926640913e-05, 5.61847846256569e-05, 6.457872223109007e-05, 6.802106508985162e-05, 7.260260463226587e-05, 7.617208029842004e-05, 6.738308729836717e-05, 6.428133929148316e-05, 8.448956214124337e-05, 6.640491483267397e-05, 5.666854849550873e-05, 5.25622526765801e-05 ] } } } } } }, "step": 8000 } ], "environment": { "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", "compile": false, "compute_capability": [ 12, 0 ], "cublas_workspace_config": ":4096:8", "cuda": "12.8", "deterministic_algorithms": true, "gpu": "NVIDIA GeForce RTX 5090", "python": "3.10.14", "torch": "2.11.0+cu128" }, "evaluations": [ { "bits_per_byte": 8.061194200882085, "cross_entropy_nats": 5.587594032287598, "step": 0 }, { "bits_per_byte": 5.191908918496252, "cross_entropy_nats": 3.598757028579712, "step": 100 }, { "bits_per_byte": 3.0881952483500545, "cross_entropy_nats": 2.1405738294124603, "step": 500 }, { "bits_per_byte": 1.948972795980338, "cross_entropy_nats": 1.3509249985218048, "step": 2000 }, { "bits_per_byte": 1.7872753095784035, "cross_entropy_nats": 1.2388448417186737, "step": 4000 }, { "bits_per_byte": 1.7030966812870931, "cross_entropy_nats": 1.1804966628551483, "step": 8000 } ], "hashes": { "final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a", "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", "final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e", "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", "initial_public_parameter_elements": 18985152, "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", "initial_public_parameter_tensors": 227, "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" }, "manifest": { "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", "input_gate_tensor_hashes": { "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" }, "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", "parent_path": "experiments/k3/attnres_gradient/manifest.json", "path": "experiments/k3/attnres_spike/manifest.json", "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" }, "model": { "aggregation_groups": 8, "blocks_per_group": 4, "d_ff": 768, "d_model": 192, "heads": 6, "layers": 32, "parameters": { "core": 18985152, "embedding": 98304, "mixer": 24960, "total": 19010112 } }, "optimizer": { "betas": [ 0.9, 0.95 ], "epsilon": 1e-08, "grad_clip": 1.0, "min_lr": 3e-05, "name": "AdamW", "peak_lr": 0.0003, "warmup_steps": 400, "weight_decay_ndim_ge_2": 0.1 }, "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", "round05_equivalence": { "checks": { "evaluations": true, "final_model_state": true, "final_optimizer_state": true, "post_mlp_element_rms_all_steps": true, "training_history": true }, "passed": true }, "run_kind": "formal", "schema_version": 1, "seed": 2026073003, "steps": 8000, "target_bytes_seen": 65536000, "timing": { "mean_ms": 224.65581085475935, "measured_steps": 7980, "median_ms": 227.5267465010984, "p95_ms": 230.52696228027344, "peak_allocated_bytes": 12841009152, "peak_reserved_bytes": 13922992128, "warmup_steps_excluded": 20 }, "training_history": [ { "bits_per_byte": 8.082576459898554, "learning_rate": 7.499999999999999e-07, "loss_nats": 5.602415084838867, "step": 1, "unclipped_grad_norm": 19.083253860473633 }, { "bits_per_byte": 7.337116283872504, "learning_rate": 7.499999999999999e-06, "loss_nats": 5.0857014656066895, "step": 10, "unclipped_grad_norm": 11.73559856414795 }, { "bits_per_byte": 6.794594947933964, "learning_rate": 1.4999999999999999e-05, "loss_nats": 4.709654331207275, "step": 20, "unclipped_grad_norm": 3.3705036640167236 }, { "bits_per_byte": 6.669727976170927, "learning_rate": 2.2499999999999998e-05, "loss_nats": 4.623103141784668, "step": 30, "unclipped_grad_norm": 3.037728786468506 }, { "bits_per_byte": 6.488515360401613, "learning_rate": 2.9999999999999997e-05, "loss_nats": 4.497496128082275, "step": 40, "unclipped_grad_norm": 3.071707010269165 }, { "bits_per_byte": 6.170372909737745, "learning_rate": 3.75e-05, "loss_nats": 4.276976585388184, "step": 50, "unclipped_grad_norm": 2.8251144886016846 }, { "bits_per_byte": 5.9458358004242795, "learning_rate": 4.4999999999999996e-05, "loss_nats": 4.121339321136475, "step": 60, "unclipped_grad_norm": 2.6051926612854004 }, { "bits_per_byte": 5.763860269615738, "learning_rate": 5.2499999999999995e-05, "loss_nats": 3.9952034950256348, "step": 70, "unclipped_grad_norm": 2.513765573501587 }, { "bits_per_byte": 5.595572150058764, "learning_rate": 5.9999999999999995e-05, "loss_nats": 3.8785550594329834, "step": 80, "unclipped_grad_norm": 2.595860719680786 }, { "bits_per_byte": 5.4051244099034195, "learning_rate": 6.749999999999999e-05, "loss_nats": 3.746546745300293, "step": 90, "unclipped_grad_norm": 2.317351818084717 }, { "bits_per_byte": 5.1779332643218385, "learning_rate": 7.5e-05, "loss_nats": 3.5890698432922363, "step": 100, "unclipped_grad_norm": 2.214414358139038 }, { "bits_per_byte": 4.995566300999934, "learning_rate": 8.249999999999999e-05, "loss_nats": 3.462662696838379, "step": 110, "unclipped_grad_norm": 2.0333311557769775 }, { "bits_per_byte": 4.799066491360726, "learning_rate": 8.999999999999999e-05, "loss_nats": 3.3264594078063965, "step": 120, "unclipped_grad_norm": 1.8019887208938599 }, { "bits_per_byte": 4.633962458601826, "learning_rate": 9.75e-05, "loss_nats": 3.2120180130004883, "step": 130, "unclipped_grad_norm": 1.6018544435501099 }, { "bits_per_byte": 4.476944362155892, "learning_rate": 0.00010499999999999999, "loss_nats": 3.1031813621520996, "step": 140, "unclipped_grad_norm": 1.3790134191513062 }, { "bits_per_byte": 4.275779695094682, "learning_rate": 0.0001125, "loss_nats": 2.963744640350342, "step": 150, "unclipped_grad_norm": 1.2061023712158203 }, { "bits_per_byte": 4.128819144071206, "learning_rate": 0.00011999999999999999, "loss_nats": 2.861879348754883, "step": 160, "unclipped_grad_norm": 1.0463906526565552 }, { "bits_per_byte": 4.021535678532382, "learning_rate": 0.00012749999999999998, "loss_nats": 2.7875161170959473, "step": 170, "unclipped_grad_norm": 0.9023508429527283 }, { "bits_per_byte": 3.919193618518149, "learning_rate": 0.00013499999999999997, "loss_nats": 2.7165780067443848, "step": 180, "unclipped_grad_norm": 0.9033810496330261 }, { "bits_per_byte": 3.8348498868820102, "learning_rate": 0.0001425, "loss_nats": 2.6581153869628906, "step": 190, "unclipped_grad_norm": 0.8499347567558289 }, { "bits_per_byte": 3.7665400977858114, "learning_rate": 0.00015, "loss_nats": 2.610766649246216, "step": 200, "unclipped_grad_norm": 0.8223105072975159 }, { "bits_per_byte": 3.6822345462981643, "learning_rate": 0.0001575, "loss_nats": 2.552330493927002, "step": 210, "unclipped_grad_norm": 0.8248683214187622 }, { "bits_per_byte": 3.6564082555833983, "learning_rate": 0.00016499999999999997, "loss_nats": 2.5344290733337402, "step": 220, "unclipped_grad_norm": 1.0457000732421875 }, { "bits_per_byte": 3.6121584953776043, "learning_rate": 0.0001725, "loss_nats": 2.5037574768066406, "step": 230, "unclipped_grad_norm": 1.2628414630889893 }, { "bits_per_byte": 3.5855720413460244, "learning_rate": 0.00017999999999999998, "loss_nats": 2.4853291511535645, "step": 240, "unclipped_grad_norm": 0.9464337229728699 }, { "bits_per_byte": 3.5024318443006237, "learning_rate": 0.0001875, "loss_nats": 2.4277007579803467, "step": 250, "unclipped_grad_norm": 0.8258991837501526 }, { "bits_per_byte": 3.520142273721174, "learning_rate": 0.000195, "loss_nats": 2.439976692199707, "step": 260, "unclipped_grad_norm": 1.1166419982910156 }, { "bits_per_byte": 3.497664141253405, "learning_rate": 0.00020249999999999996, "loss_nats": 2.42439603805542, "step": 270, "unclipped_grad_norm": 0.9347699880599976 }, { "bits_per_byte": 3.454170760746106, "learning_rate": 0.00020999999999999998, "loss_nats": 2.3942487239837646, "step": 280, "unclipped_grad_norm": 1.2552868127822876 }, { "bits_per_byte": 3.4220065653817433, "learning_rate": 0.00021749999999999997, "loss_nats": 2.3719542026519775, "step": 290, "unclipped_grad_norm": 0.8032089471817017 }, { "bits_per_byte": 3.435558454304431, "learning_rate": 0.000225, "loss_nats": 2.38134765625, "step": 300, "unclipped_grad_norm": 1.544488787651062 }, { "bits_per_byte": 3.330305071976392, "learning_rate": 0.00023249999999999996, "loss_nats": 2.308391571044922, "step": 310, "unclipped_grad_norm": 0.8757423162460327 }, { "bits_per_byte": 3.390824390953808, "learning_rate": 0.00023999999999999998, "loss_nats": 2.3503403663635254, "step": 320, "unclipped_grad_norm": 0.8823365569114685 }, { "bits_per_byte": 3.3979087003081485, "learning_rate": 0.0002475, "loss_nats": 2.355250835418701, "step": 330, "unclipped_grad_norm": 0.898759663105011 }, { "bits_per_byte": 3.325636774901372, "learning_rate": 0.00025499999999999996, "loss_nats": 2.3051557540893555, "step": 340, "unclipped_grad_norm": 0.9536095261573792 }, { "bits_per_byte": 3.3170438337336625, "learning_rate": 0.0002625, "loss_nats": 2.2991995811462402, "step": 350, "unclipped_grad_norm": 1.2654696702957153 }, { "bits_per_byte": 3.313786482326321, "learning_rate": 0.00026999999999999995, "loss_nats": 2.2969417572021484, "step": 360, "unclipped_grad_norm": 1.3773932456970215 }, { "bits_per_byte": 3.3236555347634327, "learning_rate": 0.00027749999999999997, "loss_nats": 2.3037824630737305, "step": 370, "unclipped_grad_norm": 1.2408338785171509 }, { "bits_per_byte": 3.208911117864879, "learning_rate": 0.000285, "loss_nats": 2.224247694015503, "step": 380, "unclipped_grad_norm": 1.0960922241210938 }, { "bits_per_byte": 3.2804789462990227, "learning_rate": 0.0002925, "loss_nats": 2.2738547325134277, "step": 390, "unclipped_grad_norm": 0.9866538643836975 }, { "bits_per_byte": 3.27038631641579, "learning_rate": 0.0003, "loss_nats": 2.2668590545654297, "step": 400, "unclipped_grad_norm": 1.2531954050064087 }, { "bits_per_byte": 3.2889521875417316, "learning_rate": 0.0002999988466112388, "loss_nats": 2.2797279357910156, "step": 410, "unclipped_grad_norm": 1.448500633239746 }, { "bits_per_byte": 3.205672684549132, "learning_rate": 0.00029999538646466355, "loss_nats": 2.2220029830932617, "step": 420, "unclipped_grad_norm": 0.9654834866523743 }, { "bits_per_byte": 3.218386673996816, "learning_rate": 0.0002999896196193985, "loss_nats": 2.2308156490325928, "step": 430, "unclipped_grad_norm": 0.974204421043396 }, { "bits_per_byte": 3.1822768526563587, "learning_rate": 0.00029998154617398326, "loss_nats": 2.2057862281799316, "step": 440, "unclipped_grad_norm": 1.4095590114593506 }, { "bits_per_byte": 3.237966210866943, "learning_rate": 0.0002999711662663707, "loss_nats": 2.244387149810791, "step": 450, "unclipped_grad_norm": 0.9198341369628906 }, { "bits_per_byte": 3.1857660366770637, "learning_rate": 0.0002999584800739247, "loss_nats": 2.208204746246338, "step": 460, "unclipped_grad_norm": 1.4268778562545776 }, { "bits_per_byte": 3.1576792059993566, "learning_rate": 0.0002999434878134173, "loss_nats": 2.1887364387512207, "step": 470, "unclipped_grad_norm": 1.2876354455947876 }, { "bits_per_byte": 3.131057323541699, "learning_rate": 0.000299926189741025, "loss_nats": 2.170283555984497, "step": 480, "unclipped_grad_norm": 1.0718865394592285 }, { "bits_per_byte": 3.1147361699748184, "learning_rate": 0.000299906586152324, "loss_nats": 2.158970594406128, "step": 490, "unclipped_grad_norm": 0.8680974841117859 }, { "bits_per_byte": 3.059730958749366, "learning_rate": 0.0002998846773822856, "loss_nats": 2.1208438873291016, "step": 500, "unclipped_grad_norm": 1.296195387840271 }, { "bits_per_byte": 3.12068642572937, "learning_rate": 0.0002998604638052702, "loss_nats": 2.163094997406006, "step": 510, "unclipped_grad_norm": 1.0469720363616943 }, { "bits_per_byte": 3.1397613655018657, "learning_rate": 0.0002998339458350211, "loss_nats": 2.176316738128662, "step": 520, "unclipped_grad_norm": 1.2089059352874756 }, { "bits_per_byte": 2.9844827015867583, "learning_rate": 0.0002998051239246572, "loss_nats": 2.06868577003479, "step": 530, "unclipped_grad_norm": 1.1353814601898193 }, { "bits_per_byte": 3.0015299659055814, "learning_rate": 0.00029977399856666537, "loss_nats": 2.0805020332336426, "step": 540, "unclipped_grad_norm": 0.919367253780365 }, { "bits_per_byte": 2.987599371185694, "learning_rate": 0.0002997405702928922, "loss_nats": 2.0708460807800293, "step": 550, "unclipped_grad_norm": 0.8667266368865967 }, { "bits_per_byte": 2.995183462123442, "learning_rate": 0.0002997048396745345, "loss_nats": 2.0761029720306396, "step": 560, "unclipped_grad_norm": 0.9804732203483582 }, { "bits_per_byte": 2.962294187903042, "learning_rate": 0.0002996668073221302, "loss_nats": 2.0533058643341064, "step": 570, "unclipped_grad_norm": 0.9414609670639038 }, { "bits_per_byte": 2.956716790535561, "learning_rate": 0.000299626473885547, "loss_nats": 2.0494399070739746, "step": 580, "unclipped_grad_norm": 0.8727462291717529 }, { "bits_per_byte": 2.9113505189603663, "learning_rate": 0.00029958384005397226, "loss_nats": 2.0179944038391113, "step": 590, "unclipped_grad_norm": 1.1642946004867554 }, { "bits_per_byte": 2.8665478345320485, "learning_rate": 0.0002995389065559004, "loss_nats": 1.986939549446106, "step": 600, "unclipped_grad_norm": 0.9213871359825134 }, { "bits_per_byte": 2.875505550902238, "learning_rate": 0.000299491674159121, "loss_nats": 1.9931485652923584, "step": 610, "unclipped_grad_norm": 0.7122263312339783 }, { "bits_per_byte": 2.878308696128651, "learning_rate": 0.00029944214367070535, "loss_nats": 1.9950915575027466, "step": 620, "unclipped_grad_norm": 1.4622896909713745 }, { "bits_per_byte": 2.790202156074285, "learning_rate": 0.00029939031593699284, "loss_nats": 1.934020757675171, "step": 630, "unclipped_grad_norm": 1.1472660303115845 }, { "bits_per_byte": 2.7890029210498186, "learning_rate": 0.00029933619184357654, "loss_nats": 1.9331895112991333, "step": 640, "unclipped_grad_norm": 1.1472764015197754 }, { "bits_per_byte": 2.7734051265124666, "learning_rate": 0.00029927977231528774, "loss_nats": 1.9223779439926147, "step": 650, "unclipped_grad_norm": 0.9182148575782776 }, { "bits_per_byte": 2.795219749913207, "learning_rate": 0.00029922105831618057, "loss_nats": 1.937498688697815, "step": 660, "unclipped_grad_norm": 1.083983302116394 }, { "bits_per_byte": 2.7702638633944576, "learning_rate": 0.0002991600508495153, "loss_nats": 1.9202005863189697, "step": 670, "unclipped_grad_norm": 1.1721765995025635 }, { "bits_per_byte": 2.755497089026075, "learning_rate": 0.00029909675095774114, "loss_nats": 1.9099650382995605, "step": 680, "unclipped_grad_norm": 1.280714750289917 }, { "bits_per_byte": 2.7128121989604823, "learning_rate": 0.0002990311597224787, "loss_nats": 1.8803781270980835, "step": 690, "unclipped_grad_norm": 1.0380759239196777 }, { "bits_per_byte": 2.7091302223881604, "learning_rate": 0.00029896327826450103, "loss_nats": 1.8778259754180908, "step": 700, "unclipped_grad_norm": 1.1401211023330688 }, { "bits_per_byte": 2.6870259802033667, "learning_rate": 0.0002988931077437152, "loss_nats": 1.862504482269287, "step": 710, "unclipped_grad_norm": 1.6450653076171875 }, { "bits_per_byte": 2.688485424978589, "learning_rate": 0.00029882064935914164, "loss_nats": 1.863516092300415, "step": 720, "unclipped_grad_norm": 1.0721391439437866 }, { "bits_per_byte": 2.65192277331841, "learning_rate": 0.0002987459043488943, "loss_nats": 1.8381727933883667, "step": 730, "unclipped_grad_norm": 1.0645538568496704 }, { "bits_per_byte": 2.6143492076364594, "learning_rate": 0.0002986688739901591, "loss_nats": 1.8121287822723389, "step": 740, "unclipped_grad_norm": 1.1877458095550537 }, { "bits_per_byte": 2.6277774410331767, "learning_rate": 0.0002985895595991723, "loss_nats": 1.8214365243911743, "step": 750, "unclipped_grad_norm": 1.1091431379318237 }, { "bits_per_byte": 2.540277655771055, "learning_rate": 0.00029850796253119824, "loss_nats": 1.7607862949371338, "step": 760, "unclipped_grad_norm": 1.222191333770752 }, { "bits_per_byte": 2.5790566478529025, "learning_rate": 0.00029842408418050544, "loss_nats": 1.787665843963623, "step": 770, "unclipped_grad_norm": 1.254854679107666 }, { "bits_per_byte": 2.5138998166949715, "learning_rate": 0.0002983379259803436, "loss_nats": 1.7425025701522827, "step": 780, "unclipped_grad_norm": 0.9573470950126648 }, { "bits_per_byte": 2.5258618979930834, "learning_rate": 0.0002982494894029186, "loss_nats": 1.7507940530776978, "step": 790, "unclipped_grad_norm": 1.108724594116211 }, { "bits_per_byte": 2.430956195819507, "learning_rate": 0.0002981587759593675, "loss_nats": 1.6850104331970215, "step": 800, "unclipped_grad_norm": 0.9832104444503784 }, { "bits_per_byte": 2.468076415233783, "learning_rate": 0.00029806578719973283, "loss_nats": 1.7107402086257935, "step": 810, "unclipped_grad_norm": 0.9492170214653015 }, { "bits_per_byte": 2.4602348662676934, "learning_rate": 0.00029797052471293596, "loss_nats": 1.7053048610687256, "step": 820, "unclipped_grad_norm": 1.192505121231079 }, { "bits_per_byte": 2.4583616312344954, "learning_rate": 0.0002978729901267499, "loss_nats": 1.7040064334869385, "step": 830, "unclipped_grad_norm": 0.9113606214523315 }, { "bits_per_byte": 2.4751598646748696, "learning_rate": 0.0002977731851077716, "loss_nats": 1.7156500816345215, "step": 840, "unclipped_grad_norm": 1.3671480417251587 }, { "bits_per_byte": 2.386899400147981, "learning_rate": 0.0002976711113613935, "loss_nats": 1.6544725894927979, "step": 850, "unclipped_grad_norm": 1.0324512720108032 }, { "bits_per_byte": 2.472668523994471, "learning_rate": 0.0002975667706317743, "loss_nats": 1.7139232158660889, "step": 860, "unclipped_grad_norm": 1.503003478050232 }, { "bits_per_byte": 2.4348980381772827, "learning_rate": 0.00029746016470180915, "loss_nats": 1.6877427101135254, "step": 870, "unclipped_grad_norm": 1.1144746541976929 }, { "bits_per_byte": 2.3709108610278693, "learning_rate": 0.00029735129539309926, "loss_nats": 1.64339017868042, "step": 880, "unclipped_grad_norm": 1.0134795904159546 }, { "bits_per_byte": 2.3377405671559597, "learning_rate": 0.0002972401645659207, "loss_nats": 1.6203982830047607, "step": 890, "unclipped_grad_norm": 1.0625330209732056 }, { "bits_per_byte": 2.389408283058768, "learning_rate": 0.00029712677411919283, "loss_nats": 1.6562116146087646, "step": 900, "unclipped_grad_norm": 0.9719302654266357 }, { "bits_per_byte": 2.3782030974071033, "learning_rate": 0.0002970111259904454, "loss_nats": 1.6484447717666626, "step": 910, "unclipped_grad_norm": 1.0672563314437866 }, { "bits_per_byte": 2.3530695528100125, "learning_rate": 0.0002968932221557859, "loss_nats": 1.6310235261917114, "step": 920, "unclipped_grad_norm": 1.1197935342788696 }, { "bits_per_byte": 2.297453115243394, "learning_rate": 0.0002967730646298655, "loss_nats": 1.5924731492996216, "step": 930, "unclipped_grad_norm": 1.0027272701263428 }, { "bits_per_byte": 2.344061961471073, "learning_rate": 0.0002966506554658449, "loss_nats": 1.6247799396514893, "step": 940, "unclipped_grad_norm": 1.0651428699493408 }, { "bits_per_byte": 2.367935217202641, "learning_rate": 0.00029652599675535876, "loss_nats": 1.6413276195526123, "step": 950, "unclipped_grad_norm": 0.9822106957435608 }, { "bits_per_byte": 2.309206925551311, "learning_rate": 0.0002963990906284808, "loss_nats": 1.6006202697753906, "step": 960, "unclipped_grad_norm": 0.8860561847686768 }, { "bits_per_byte": 2.2419830349126597, "learning_rate": 0.0002962699392536863, "loss_nats": 1.5540242195129395, "step": 970, "unclipped_grad_norm": 1.1816905736923218 }, { "bits_per_byte": 2.3851068249780383, "learning_rate": 0.0002961385448378161, "loss_nats": 1.65323007106781, "step": 980, "unclipped_grad_norm": 0.9257189035415649 }, { "bits_per_byte": 2.232579711494085, "learning_rate": 0.00029600490962603817, "loss_nats": 1.547506332397461, "step": 990, "unclipped_grad_norm": 0.9634203910827637 }, { "bits_per_byte": 2.2247954327507324, "learning_rate": 0.00029586903590180956, "loss_nats": 1.5421106815338135, "step": 1000, "unclipped_grad_norm": 0.809658944606781 }, { "bits_per_byte": 2.2354428786656486, "learning_rate": 0.00029573092598683735, "loss_nats": 1.5494909286499023, "step": 1010, "unclipped_grad_norm": 1.3530596494674683 }, { "bits_per_byte": 2.2067197121797495, "learning_rate": 0.00029559058224103895, "loss_nats": 1.5295815467834473, "step": 1020, "unclipped_grad_norm": 1.0303069353103638 }, { "bits_per_byte": 2.1954008459958647, "learning_rate": 0.0002954480070625017, "loss_nats": 1.5217359066009521, "step": 1030, "unclipped_grad_norm": 0.9071640968322754 }, { "bits_per_byte": 2.291673982226451, "learning_rate": 0.00029530320288744205, "loss_nats": 1.5884673595428467, "step": 1040, "unclipped_grad_norm": 1.3239798545837402 }, { "bits_per_byte": 2.196653911590051, "learning_rate": 0.0002951561721901638, "loss_nats": 1.5226044654846191, "step": 1050, "unclipped_grad_norm": 0.9085557460784912 }, { "bits_per_byte": 2.225028985190604, "learning_rate": 0.00029500691748301594, "loss_nats": 1.5422725677490234, "step": 1060, "unclipped_grad_norm": 0.9488101005554199 }, { "bits_per_byte": 2.275690258620563, "learning_rate": 0.00029485544131634963, "loss_nats": 1.5773882865905762, "step": 1070, "unclipped_grad_norm": 1.047879695892334 }, { "bits_per_byte": 2.235951603346901, "learning_rate": 0.0002947017462784745, "loss_nats": 1.5498435497283936, "step": 1080, "unclipped_grad_norm": 0.8919407725334167 }, { "bits_per_byte": 2.1733449309359636, "learning_rate": 0.00029454583499561476, "loss_nats": 1.5064479112625122, "step": 1090, "unclipped_grad_norm": 1.0036922693252563 }, { "bits_per_byte": 2.169043472855234, "learning_rate": 0.00029438771013186413, "loss_nats": 1.5034663677215576, "step": 1100, "unclipped_grad_norm": 0.9904995560646057 }, { "bits_per_byte": 2.124962083664635, "learning_rate": 0.0002942273743891401, "loss_nats": 1.4729114770889282, "step": 1110, "unclipped_grad_norm": 0.9176430702209473 }, { "bits_per_byte": 2.176794214981668, "learning_rate": 0.00029406483050713824, "loss_nats": 1.5088387727737427, "step": 1120, "unclipped_grad_norm": 0.9229135513305664 }, { "bits_per_byte": 2.210271497885368, "learning_rate": 0.00029390008126328504, "loss_nats": 1.53204345703125, "step": 1130, "unclipped_grad_norm": 0.871993899345398 }, { "bits_per_byte": 2.123456375556331, "learning_rate": 0.00029373312947269045, "loss_nats": 1.4718677997589111, "step": 1140, "unclipped_grad_norm": 0.8673478960990906 }, { "bits_per_byte": 2.1911007637763418, "learning_rate": 0.00029356397798809995, "loss_nats": 1.518755316734314, "step": 1150, "unclipped_grad_norm": 0.8959671258926392 }, { "bits_per_byte": 2.0953000638876755, "learning_rate": 0.0002933926296998457, "loss_nats": 1.4523513317108154, "step": 1160, "unclipped_grad_norm": 0.9113589525222778 }, { "bits_per_byte": 2.1570418355474237, "learning_rate": 0.0002932190875357972, "loss_nats": 1.495147466659546, "step": 1170, "unclipped_grad_norm": 0.8980047702789307 }, { "bits_per_byte": 2.1555593450969863, "learning_rate": 0.00029304335446131126, "loss_nats": 1.4941198825836182, "step": 1180, "unclipped_grad_norm": 0.948503315448761 }, { "bits_per_byte": 2.1334348089593913, "learning_rate": 0.00029286543347918116, "loss_nats": 1.4787843227386475, "step": 1190, "unclipped_grad_norm": 1.1094794273376465 }, { "bits_per_byte": 2.1317098229712372, "learning_rate": 0.0002926853276295856, "loss_nats": 1.4775886535644531, "step": 1200, "unclipped_grad_norm": 0.8304017782211304 }, { "bits_per_byte": 2.1385674591917407, "learning_rate": 0.0002925030399900368, "loss_nats": 1.482342004776001, "step": 1210, "unclipped_grad_norm": 0.8620252013206482 }, { "bits_per_byte": 2.0980762078379027, "learning_rate": 0.00029231857367532726, "loss_nats": 1.4542756080627441, "step": 1220, "unclipped_grad_norm": 0.88591068983078 }, { "bits_per_byte": 2.108927281211441, "learning_rate": 0.0002921319318374776, "loss_nats": 1.4617969989776611, "step": 1230, "unclipped_grad_norm": 1.0479243993759155 }, { "bits_per_byte": 2.081558994049039, "learning_rate": 0.00029194311766568174, "loss_nats": 1.442826747894287, "step": 1240, "unclipped_grad_norm": 0.9365729689598083 }, { "bits_per_byte": 2.0172010223043655, "learning_rate": 0.000291752134386253, "loss_nats": 1.3982172012329102, "step": 1250, "unclipped_grad_norm": 0.8966301083564758 }, { "bits_per_byte": 2.0690183631134245, "learning_rate": 0.00029155898526256856, "loss_nats": 1.4341342449188232, "step": 1260, "unclipped_grad_norm": 0.8974369168281555 }, { "bits_per_byte": 2.101840220134686, "learning_rate": 0.0002913636735950141, "loss_nats": 1.4568846225738525, "step": 1270, "unclipped_grad_norm": 0.9220007061958313 }, { "bits_per_byte": 2.1256708241688407, "learning_rate": 0.0002911662027209272, "loss_nats": 1.473402738571167, "step": 1280, "unclipped_grad_norm": 0.9313441514968872 }, { "bits_per_byte": 2.076867651298803, "learning_rate": 0.00029096657601454013, "loss_nats": 1.439574956893921, "step": 1290, "unclipped_grad_norm": 0.8451681137084961 }, { "bits_per_byte": 2.032476865319302, "learning_rate": 0.00029076479688692266, "loss_nats": 1.4088056087493896, "step": 1300, "unclipped_grad_norm": 0.9314314126968384 }, { "bits_per_byte": 2.058903719450881, "learning_rate": 0.00029056086878592323, "loss_nats": 1.4271233081817627, "step": 1310, "unclipped_grad_norm": 0.8796194195747375 }, { "bits_per_byte": 2.0275063947267156, "learning_rate": 0.00029035479519611045, "loss_nats": 1.4053603410720825, "step": 1320, "unclipped_grad_norm": 0.9140937924385071 }, { "bits_per_byte": 2.037489299678698, "learning_rate": 0.0002901465796387134, "loss_nats": 1.4122799634933472, "step": 1330, "unclipped_grad_norm": 0.821657657623291 }, { "bits_per_byte": 2.0199990080512524, "learning_rate": 0.00028993622567156154, "loss_nats": 1.4001566171646118, "step": 1340, "unclipped_grad_norm": 0.8321346044540405 }, { "bits_per_byte": 2.048903616233812, "learning_rate": 0.0002897237368890237, "loss_nats": 1.420191764831543, "step": 1350, "unclipped_grad_norm": 0.9421875476837158 }, { "bits_per_byte": 2.00944598061166, "learning_rate": 0.0002895091169219469, "loss_nats": 1.3928418159484863, "step": 1360, "unclipped_grad_norm": 0.8682821393013 }, { "bits_per_byte": 2.0435301902902543, "learning_rate": 0.00028929236943759425, "loss_nats": 1.4164671897888184, "step": 1370, "unclipped_grad_norm": 0.8776147961616516 }, { "bits_per_byte": 2.0135448431296727, "learning_rate": 0.00028907349813958225, "loss_nats": 1.39568293094635, "step": 1380, "unclipped_grad_norm": 0.87440025806427 }, { "bits_per_byte": 2.0533221944997795, "learning_rate": 0.0002888525067678175, "loss_nats": 1.4232544898986816, "step": 1390, "unclipped_grad_norm": 0.7801422476768494 }, { "bits_per_byte": 2.1564656936670334, "learning_rate": 0.00028862939909843273, "loss_nats": 1.4947481155395508, "step": 1400, "unclipped_grad_norm": 0.8561526536941528 }, { "bits_per_byte": 2.0373921294809607, "learning_rate": 0.0002884041789437225, "loss_nats": 1.412212610244751, "step": 1410, "unclipped_grad_norm": 0.8376067876815796 }, { "bits_per_byte": 2.0325960492963495, "learning_rate": 0.0002881768501520778, "loss_nats": 1.4088882207870483, "step": 1420, "unclipped_grad_norm": 0.751315712928772 }, { "bits_per_byte": 1.9860154082234123, "learning_rate": 0.0002879474166079205, "loss_nats": 1.376600980758667, "step": 1430, "unclipped_grad_norm": 0.9057046175003052 }, { "bits_per_byte": 2.015529006972677, "learning_rate": 0.00028771588223163694, "loss_nats": 1.3970582485198975, "step": 1440, "unclipped_grad_norm": 0.8342951536178589 }, { "bits_per_byte": 1.9818307263626185, "learning_rate": 0.00028748225097951054, "loss_nats": 1.3737003803253174, "step": 1450, "unclipped_grad_norm": 0.8142686486244202 }, { "bits_per_byte": 2.1060192265680016, "learning_rate": 0.000287246526843655, "loss_nats": 1.459781289100647, "step": 1460, "unclipped_grad_norm": 0.8357279896736145 }, { "bits_per_byte": 2.079546797033944, "learning_rate": 0.00028700871385194533, "loss_nats": 1.441431999206543, "step": 1470, "unclipped_grad_norm": 0.7720605731010437 }, { "bits_per_byte": 2.025708316111945, "learning_rate": 0.00028676881606794945, "loss_nats": 1.404114007949829, "step": 1480, "unclipped_grad_norm": 0.8328851461410522 }, { "bits_per_byte": 2.0422327531721454, "learning_rate": 0.00028652683759085854, "loss_nats": 1.4155678749084473, "step": 1490, "unclipped_grad_norm": 0.9698525071144104 }, { "bits_per_byte": 1.9217496191796013, "learning_rate": 0.0002862827825554171, "loss_nats": 1.3320553302764893, "step": 1500, "unclipped_grad_norm": 0.7396577000617981 }, { "bits_per_byte": 1.9436662282923034, "learning_rate": 0.00028603665513185245, "loss_nats": 1.347246766090393, "step": 1510, "unclipped_grad_norm": 0.809975266456604 }, { "bits_per_byte": 1.9993110429963143, "learning_rate": 0.000285788459525803, "loss_nats": 1.3858168125152588, "step": 1520, "unclipped_grad_norm": 0.8165730834007263 }, { "bits_per_byte": 2.03089187320895, "learning_rate": 0.00028553819997824707, "loss_nats": 1.4077069759368896, "step": 1530, "unclipped_grad_norm": 0.7620540261268616 }, { "bits_per_byte": 2.002577509484152, "learning_rate": 0.00028528588076542964, "loss_nats": 1.3880809545516968, "step": 1540, "unclipped_grad_norm": 0.8928612470626831 }, { "bits_per_byte": 2.0629018001354886, "learning_rate": 0.00028503150619878996, "loss_nats": 1.4298945665359497, "step": 1550, "unclipped_grad_norm": 0.7963082194328308 }, { "bits_per_byte": 1.9429003895480113, "learning_rate": 0.0002847750806248876, "loss_nats": 1.3467159271240234, "step": 1560, "unclipped_grad_norm": 0.7710158228874207 }, { "bits_per_byte": 1.9502758655302628, "learning_rate": 0.000284516608425328, "loss_nats": 1.3518282175064087, "step": 1570, "unclipped_grad_norm": 0.8057935833930969 }, { "bits_per_byte": 1.9693647358974786, "learning_rate": 0.000284256094016688, "loss_nats": 1.3650596141815186, "step": 1580, "unclipped_grad_norm": 0.8588574528694153 }, { "bits_per_byte": 1.972771368245769, "learning_rate": 0.0002839935418504399, "loss_nats": 1.3674209117889404, "step": 1590, "unclipped_grad_norm": 0.8608299493789673 }, { "bits_per_byte": 1.9610373359427014, "learning_rate": 0.000283728956412876, "loss_nats": 1.3592875003814697, "step": 1600, "unclipped_grad_norm": 0.8587453365325928 }, { "bits_per_byte": 1.9257736692444893, "learning_rate": 0.00028346234222503136, "loss_nats": 1.3348445892333984, "step": 1610, "unclipped_grad_norm": 0.8028180003166199 }, { "bits_per_byte": 1.929578785394829, "learning_rate": 0.00028319370384260675, "loss_nats": 1.3374820947647095, "step": 1620, "unclipped_grad_norm": 0.7991352677345276 }, { "bits_per_byte": 1.9716548568763677, "learning_rate": 0.00028292304585589105, "loss_nats": 1.3666470050811768, "step": 1630, "unclipped_grad_norm": 0.7573540806770325 }, { "bits_per_byte": 1.9236465878186182, "learning_rate": 0.0002826503728896824, "loss_nats": 1.3333702087402344, "step": 1640, "unclipped_grad_norm": 0.8524133563041687 }, { "bits_per_byte": 1.9335809936630575, "learning_rate": 0.0002823756896032095, "loss_nats": 1.3402562141418457, "step": 1650, "unclipped_grad_norm": 0.7908568978309631 }, { "bits_per_byte": 1.9524230689262851, "learning_rate": 0.0002820990006900518, "loss_nats": 1.3533165454864502, "step": 1660, "unclipped_grad_norm": 0.8385109901428223 }, { "bits_per_byte": 1.8864807930018634, "learning_rate": 0.00028182031087805947, "loss_nats": 1.3076088428497314, "step": 1670, "unclipped_grad_norm": 0.8104477524757385 }, { "bits_per_byte": 1.918672677573015, "learning_rate": 0.0002815396249292725, "loss_nats": 1.3299225568771362, "step": 1680, "unclipped_grad_norm": 0.7970330119132996 }, { "bits_per_byte": 1.9517956762159374, "learning_rate": 0.0002812569476398391, "loss_nats": 1.352881669998169, "step": 1690, "unclipped_grad_norm": 0.7619603276252747 }, { "bits_per_byte": 1.8990867773448057, "learning_rate": 0.00028097228383993446, "loss_nats": 1.3163466453552246, "step": 1700, "unclipped_grad_norm": 0.8769286274909973 }, { "bits_per_byte": 1.938959579086141, "learning_rate": 0.0002806856383936774, "loss_nats": 1.3439843654632568, "step": 1710, "unclipped_grad_norm": 0.7246651649475098 }, { "bits_per_byte": 1.847509524316351, "learning_rate": 0.00028039701619904774, "loss_nats": 1.2805960178375244, "step": 1720, "unclipped_grad_norm": 0.7415109276771545 }, { "bits_per_byte": 1.944612132872049, "learning_rate": 0.0002801064221878024, "loss_nats": 1.3479024171829224, "step": 1730, "unclipped_grad_norm": 0.8057509660720825 }, { "bits_per_byte": 1.956270664791388, "learning_rate": 0.00027981386132539134, "loss_nats": 1.3559834957122803, "step": 1740, "unclipped_grad_norm": 0.8188740611076355 }, { "bits_per_byte": 1.908994697861012, "learning_rate": 0.0002795193386108724, "loss_nats": 1.3232142925262451, "step": 1750, "unclipped_grad_norm": 0.7595267295837402 }, { "bits_per_byte": 1.9438418225788343, "learning_rate": 0.00027922285907682625, "loss_nats": 1.3473684787750244, "step": 1760, "unclipped_grad_norm": 0.7832275629043579 }, { "bits_per_byte": 1.9277638524802738, "learning_rate": 0.00027892442778927005, "loss_nats": 1.33622407913208, "step": 1770, "unclipped_grad_norm": 0.7276267409324646 }, { "bits_per_byte": 1.987644083927083, "learning_rate": 0.00027862404984757116, "loss_nats": 1.377729892730713, "step": 1780, "unclipped_grad_norm": 0.7699368000030518 }, { "bits_per_byte": 1.8704368754681737, "learning_rate": 0.0002783217303843597, "loss_nats": 1.2964880466461182, "step": 1790, "unclipped_grad_norm": 0.7781511545181274 }, { "bits_per_byte": 2.006532250540742, "learning_rate": 0.00027801747456544134, "loss_nats": 1.390822172164917, "step": 1800, "unclipped_grad_norm": 0.8113785982131958 }, { "bits_per_byte": 1.8832052114335305, "learning_rate": 0.00027771128758970843, "loss_nats": 1.3053383827209473, "step": 1810, "unclipped_grad_norm": 0.8088275790214539 }, { "bits_per_byte": 1.9263209180395349, "learning_rate": 0.00027740317468905165, "loss_nats": 1.335223913192749, "step": 1820, "unclipped_grad_norm": 0.7881855964660645 }, { "bits_per_byte": 1.968589438107389, "learning_rate": 0.0002770931411282704, "loss_nats": 1.3645222187042236, "step": 1830, "unclipped_grad_norm": 0.7948311567306519 }, { "bits_per_byte": 1.9398172565659937, "learning_rate": 0.0002767811922049827, "loss_nats": 1.3445788621902466, "step": 1840, "unclipped_grad_norm": 0.8343047499656677 }, { "bits_per_byte": 1.8747907882574166, "learning_rate": 0.00027646733324953503, "loss_nats": 1.2995059490203857, "step": 1850, "unclipped_grad_norm": 0.7942258715629578 }, { "bits_per_byte": 1.9135037390013332, "learning_rate": 0.00027615156962491097, "loss_nats": 1.3263397216796875, "step": 1860, "unclipped_grad_norm": 0.769598126411438 }, { "bits_per_byte": 1.987024774401326, "learning_rate": 0.0002758339067266396, "loss_nats": 1.3773006200790405, "step": 1870, "unclipped_grad_norm": 0.7647593021392822 }, { "bits_per_byte": 1.8769319722606586, "learning_rate": 0.0002755143499827035, "loss_nats": 1.300990104675293, "step": 1880, "unclipped_grad_norm": 0.8446611166000366 }, { "bits_per_byte": 1.8953207012562119, "learning_rate": 0.0002751929048534456, "loss_nats": 1.3137362003326416, "step": 1890, "unclipped_grad_norm": 0.7048609852790833 }, { "bits_per_byte": 1.915848206497895, "learning_rate": 0.0002748695768314763, "loss_nats": 1.3279647827148438, "step": 1900, "unclipped_grad_norm": 0.7416327595710754 }, { "bits_per_byte": 1.9362319342434569, "learning_rate": 0.00027454437144157947, "loss_nats": 1.3420937061309814, "step": 1910, "unclipped_grad_norm": 0.8084686398506165 }, { "bits_per_byte": 1.8860243510464736, "learning_rate": 0.00027421729424061787, "loss_nats": 1.3072924613952637, "step": 1920, "unclipped_grad_norm": 0.8338462710380554 }, { "bits_per_byte": 1.930916638435891, "learning_rate": 0.0002738883508174385, "loss_nats": 1.338409423828125, "step": 1930, "unclipped_grad_norm": 0.6878981590270996 }, { "bits_per_byte": 1.9257588787365152, "learning_rate": 0.0002735575467927768, "loss_nats": 1.334834337234497, "step": 1940, "unclipped_grad_norm": 0.6981199383735657 }, { "bits_per_byte": 1.8699291826828268, "learning_rate": 0.0002732248878191609, "loss_nats": 1.2961361408233643, "step": 1950, "unclipped_grad_norm": 0.76193767786026 }, { "bits_per_byte": 1.859464038377825, "learning_rate": 0.0002728903795808148, "loss_nats": 1.2888822555541992, "step": 1960, "unclipped_grad_norm": 0.7367894053459167 }, { "bits_per_byte": 1.8691335909399354, "learning_rate": 0.0002725540277935614, "loss_nats": 1.2955846786499023, "step": 1970, "unclipped_grad_norm": 0.8052698373794556 }, { "bits_per_byte": 1.8850715671606937, "learning_rate": 0.00027221583820472456, "loss_nats": 1.3066320419311523, "step": 1980, "unclipped_grad_norm": 0.7624203562736511 }, { "bits_per_byte": 1.8715028239382214, "learning_rate": 0.0002718758165930314, "loss_nats": 1.297226905822754, "step": 1990, "unclipped_grad_norm": 0.779162585735321 }, { "bits_per_byte": 1.9325518494802945, "learning_rate": 0.0002715339687685131, "loss_nats": 1.3395428657531738, "step": 2000, "unclipped_grad_norm": 0.7498615384101868 }, { "bits_per_byte": 1.9335421255839624, "learning_rate": 0.00027119030057240576, "loss_nats": 1.3402292728424072, "step": 2010, "unclipped_grad_norm": 0.7343811392784119 }, { "bits_per_byte": 1.838237939608337, "learning_rate": 0.0002708448178770506, "loss_nats": 1.2741694450378418, "step": 2020, "unclipped_grad_norm": 0.7684847116470337 }, { "bits_per_byte": 1.8631659649376469, "learning_rate": 0.00027049752658579376, "loss_nats": 1.2914482355117798, "step": 2030, "unclipped_grad_norm": 0.6878870129585266 }, { "bits_per_byte": 1.8841837927169398, "learning_rate": 0.00027014843263288514, "loss_nats": 1.3060166835784912, "step": 2040, "unclipped_grad_norm": 0.853155255317688 }, { "bits_per_byte": 1.8339112000779305, "learning_rate": 0.00026979754198337724, "loss_nats": 1.2711703777313232, "step": 2050, "unclipped_grad_norm": 0.8042898774147034 }, { "bits_per_byte": 1.8136427007084468, "learning_rate": 0.00026944486063302305, "loss_nats": 1.2571213245391846, "step": 2060, "unclipped_grad_norm": 0.7331950664520264 }, { "bits_per_byte": 1.9306452498128293, "learning_rate": 0.0002690903946081738, "loss_nats": 1.3382213115692139, "step": 2070, "unclipped_grad_norm": 0.7544640898704529 }, { "bits_per_byte": 1.9159075405124426, "learning_rate": 0.00026873414996567596, "loss_nats": 1.3280059099197388, "step": 2080, "unclipped_grad_norm": 0.7521310448646545 }, { "bits_per_byte": 1.8394872215842044, "learning_rate": 0.00026837613279276736, "loss_nats": 1.2750353813171387, "step": 2090, "unclipped_grad_norm": 0.7430872321128845 }, { "bits_per_byte": 1.7890550290451945, "learning_rate": 0.0002680163492069739, "loss_nats": 1.2400784492492676, "step": 2100, "unclipped_grad_norm": 0.7162277698516846 }, { "bits_per_byte": 1.8155914361253733, "learning_rate": 0.00026765480535600414, "loss_nats": 1.2584720849990845, "step": 2110, "unclipped_grad_norm": 0.7056337594985962 }, { "bits_per_byte": 1.834290249840432, "learning_rate": 0.0002672915074176452, "loss_nats": 1.2714331150054932, "step": 2120, "unclipped_grad_norm": 0.8666519522666931 }, { "bits_per_byte": 1.8865812308699674, "learning_rate": 0.00026692646159965624, "loss_nats": 1.307678461074829, "step": 2130, "unclipped_grad_norm": 0.762921154499054 }, { "bits_per_byte": 1.9030090136803828, "learning_rate": 0.0002665596741396633, "loss_nats": 1.3190653324127197, "step": 2140, "unclipped_grad_norm": 0.7508745789527893 }, { "bits_per_byte": 1.8090877402003454, "learning_rate": 0.000266191151305052, "loss_nats": 1.2539640665054321, "step": 2150, "unclipped_grad_norm": 0.7472740411758423 }, { "bits_per_byte": 1.898260228724759, "learning_rate": 0.00026582089939286075, "loss_nats": 1.3157737255096436, "step": 2160, "unclipped_grad_norm": 0.7288866639137268 }, { "bits_per_byte": 1.8098697453138184, "learning_rate": 0.0002654489247296732, "loss_nats": 1.2545061111450195, "step": 2170, "unclipped_grad_norm": 0.7270416617393494 }, { "bits_per_byte": 1.896411071262682, "learning_rate": 0.00026507523367151007, "loss_nats": 1.3144919872283936, "step": 2180, "unclipped_grad_norm": 0.7397056221961975 }, { "bits_per_byte": 1.8517294626205718, "learning_rate": 0.00026469983260372037, "loss_nats": 1.283521056175232, "step": 2190, "unclipped_grad_norm": 0.7186560034751892 }, { "bits_per_byte": 1.8299633383273748, "learning_rate": 0.0002643227279408727, "loss_nats": 1.268433928489685, "step": 2200, "unclipped_grad_norm": 0.6916610598564148 }, { "bits_per_byte": 1.8274437924922347, "learning_rate": 0.0002639439261266453, "loss_nats": 1.2666875123977661, "step": 2210, "unclipped_grad_norm": 0.7494430541992188 }, { "bits_per_byte": 1.8262679471082857, "learning_rate": 0.000263563433633716, "loss_nats": 1.2658724784851074, "step": 2220, "unclipped_grad_norm": 0.782909095287323 }, { "bits_per_byte": 1.782249847533204, "learning_rate": 0.00026318125696365156, "loss_nats": 1.2353614568710327, "step": 2230, "unclipped_grad_norm": 0.7370295524597168 }, { "bits_per_byte": 1.8783687153259665, "learning_rate": 0.000262797402646797, "loss_nats": 1.3019859790802002, "step": 2240, "unclipped_grad_norm": 0.7077252268791199 }, { "bits_per_byte": 1.844123529886164, "learning_rate": 0.00026241187724216334, "loss_nats": 1.2782490253448486, "step": 2250, "unclipped_grad_norm": 0.7420696020126343 }, { "bits_per_byte": 1.897104505310961, "learning_rate": 0.0002620246873373162, "loss_nats": 1.3149726390838623, "step": 2260, "unclipped_grad_norm": 0.7444695234298706 }, { "bits_per_byte": 1.7929542197055561, "learning_rate": 0.00026163583954826264, "loss_nats": 1.242781162261963, "step": 2270, "unclipped_grad_norm": 0.7164454460144043 }, { "bits_per_byte": 1.8643507534194408, "learning_rate": 0.0002612453405193386, "loss_nats": 1.2922694683074951, "step": 2280, "unclipped_grad_norm": 0.7152382135391235 }, { "bits_per_byte": 1.8035947863144381, "learning_rate": 0.000260853196923095, "loss_nats": 1.2501566410064697, "step": 2290, "unclipped_grad_norm": 0.7178454399108887 }, { "bits_per_byte": 1.7935200426268947, "learning_rate": 0.0002604594154601839, "loss_nats": 1.243173360824585, "step": 2300, "unclipped_grad_norm": 0.7068437337875366 }, { "bits_per_byte": 1.8020768674379228, "learning_rate": 0.0002600640028592441, "loss_nats": 1.2491044998168945, "step": 2310, "unclipped_grad_norm": 0.7017168998718262 }, { "bits_per_byte": 1.8564474626816911, "learning_rate": 0.0002596669658767859, "loss_nats": 1.2867913246154785, "step": 2320, "unclipped_grad_norm": 0.6947462558746338 }, { "bits_per_byte": 1.7936961528613782, "learning_rate": 0.000259268311297076, "loss_nats": 1.243295431137085, "step": 2330, "unclipped_grad_norm": 0.6949398517608643 }, { "bits_per_byte": 1.8129625093242843, "learning_rate": 0.00025886804593202114, "loss_nats": 1.2566498517990112, "step": 2340, "unclipped_grad_norm": 0.7047421336174011 }, { "bits_per_byte": 1.871945851246844, "learning_rate": 0.00025846617662105193, "loss_nats": 1.2975339889526367, "step": 2350, "unclipped_grad_norm": 0.7291849851608276 }, { "bits_per_byte": 1.7987307729827362, "learning_rate": 0.00025806271023100637, "loss_nats": 1.2467851638793945, "step": 2360, "unclipped_grad_norm": 0.7353813052177429 }, { "bits_per_byte": 1.7311453748119714, "learning_rate": 0.00025765765365601165, "loss_nats": 1.1999385356903076, "step": 2370, "unclipped_grad_norm": 0.6746760010719299 }, { "bits_per_byte": 1.834468595849377, "learning_rate": 0.00025725101381736707, "loss_nats": 1.2715567350387573, "step": 2380, "unclipped_grad_norm": 0.7166123390197754 }, { "bits_per_byte": 1.8260691351638882, "learning_rate": 0.00025684279766342567, "loss_nats": 1.2657346725463867, "step": 2390, "unclipped_grad_norm": 0.6613916158676147 }, { "bits_per_byte": 1.8641103216735346, "learning_rate": 0.00025643301216947504, "loss_nats": 1.2921028137207031, "step": 2400, "unclipped_grad_norm": 0.824955403804779 }, { "bits_per_byte": 1.793840446305452, "learning_rate": 0.00025602166433761866, "loss_nats": 1.243395447731018, "step": 2410, "unclipped_grad_norm": 0.7929511070251465 }, { "bits_per_byte": 1.8233349549804712, "learning_rate": 0.0002556087611966561, "loss_nats": 1.2638394832611084, "step": 2420, "unclipped_grad_norm": 0.725745677947998 }, { "bits_per_byte": 1.825773496987055, "learning_rate": 0.00025519430980196277, "loss_nats": 1.265529751777649, "step": 2430, "unclipped_grad_norm": 0.7284960746765137 }, { "bits_per_byte": 1.7725216488871491, "learning_rate": 0.0002547783172353695, "loss_nats": 1.2286183834075928, "step": 2440, "unclipped_grad_norm": 0.7458904981613159 }, { "bits_per_byte": 1.9001444706476118, "learning_rate": 0.0002543607906050415, "loss_nats": 1.317079782485962, "step": 2450, "unclipped_grad_norm": 0.8321223258972168 }, { "bits_per_byte": 1.7654523020234334, "learning_rate": 0.0002539417370453568, "loss_nats": 1.223718285560608, "step": 2460, "unclipped_grad_norm": 0.7338651418685913 }, { "bits_per_byte": 1.7862824967306352, "learning_rate": 0.00025352116371678464, "loss_nats": 1.2381566762924194, "step": 2470, "unclipped_grad_norm": 0.7279908061027527 }, { "bits_per_byte": 1.8666427662074894, "learning_rate": 0.0002530990778057627, "loss_nats": 1.2938581705093384, "step": 2480, "unclipped_grad_norm": 0.7701565027236938 }, { "bits_per_byte": 1.850673661126925, "learning_rate": 0.00025267548652457475, "loss_nats": 1.2827892303466797, "step": 2490, "unclipped_grad_norm": 0.6776601672172546 }, { "bits_per_byte": 1.8000481600883451, "learning_rate": 0.00025225039711122696, "loss_nats": 1.2476983070373535, "step": 2500, "unclipped_grad_norm": 0.7444605231285095 }, { "bits_per_byte": 1.7603095048146828, "learning_rate": 0.00025182381682932464, "loss_nats": 1.220153570175171, "step": 2510, "unclipped_grad_norm": 0.7512273788452148 }, { "bits_per_byte": 1.7805241736144466, "learning_rate": 0.0002513957529679477, "loss_nats": 1.2341653108596802, "step": 2520, "unclipped_grad_norm": 0.6968159079551697 }, { "bits_per_byte": 1.803689032807111, "learning_rate": 0.00025096621284152656, "loss_nats": 1.2502219676971436, "step": 2530, "unclipped_grad_norm": 0.7150991559028625 }, { "bits_per_byte": 1.8049077018711248, "learning_rate": 0.0002505352037897168, "loss_nats": 1.2510666847229004, "step": 2540, "unclipped_grad_norm": 0.7343016862869263 }, { "bits_per_byte": 1.742795479581418, "learning_rate": 0.0002501027331772739, "loss_nats": 1.2080137729644775, "step": 2550, "unclipped_grad_norm": 0.7049802541732788 }, { "bits_per_byte": 1.7179068382791558, "learning_rate": 0.00024966880839392746, "loss_nats": 1.1907622814178467, "step": 2560, "unclipped_grad_norm": 0.7055768370628357 }, { "bits_per_byte": 1.7731971967397384, "learning_rate": 0.0002492334368542548, "loss_nats": 1.2290866374969482, "step": 2570, "unclipped_grad_norm": 0.6996883749961853 }, { "bits_per_byte": 1.7458676056737803, "learning_rate": 0.00024879662599755416, "loss_nats": 1.2101432085037231, "step": 2580, "unclipped_grad_norm": 0.6933735609054565 }, { "bits_per_byte": 1.7458146350173145, "learning_rate": 0.00024835838328771803, "loss_nats": 1.2101064920425415, "step": 2590, "unclipped_grad_norm": 0.7094681859016418 }, { "bits_per_byte": 1.7637799427264436, "learning_rate": 0.0002479187162131051, "loss_nats": 1.2225590944290161, "step": 2600, "unclipped_grad_norm": 0.7199745178222656 }, { "bits_per_byte": 1.786311561798631, "learning_rate": 0.00024747763228641264, "loss_nats": 1.2381768226623535, "step": 2610, "unclipped_grad_norm": 0.7421150207519531 }, { "bits_per_byte": 1.7252271078304804, "learning_rate": 0.00024703513904454795, "loss_nats": 1.1958363056182861, "step": 2620, "unclipped_grad_norm": 0.7062536478042603 }, { "bits_per_byte": 1.7243496523457782, "learning_rate": 0.00024659124404849956, "loss_nats": 1.195228099822998, "step": 2630, "unclipped_grad_norm": 0.6937817931175232 }, { "bits_per_byte": 1.7124126805146924, "learning_rate": 0.00024614595488320833, "loss_nats": 1.1869540214538574, "step": 2640, "unclipped_grad_norm": 0.7252171039581299 }, { "bits_per_byte": 1.781446344588373, "learning_rate": 0.00024569927915743745, "loss_nats": 1.2348045110702515, "step": 2650, "unclipped_grad_norm": 0.6726356744766235 }, { "bits_per_byte": 1.6915262474806587, "learning_rate": 0.0002452512245036427, "loss_nats": 1.1724766492843628, "step": 2660, "unclipped_grad_norm": 0.6618354320526123 }, { "bits_per_byte": 1.7563157236963467, "learning_rate": 0.00024480179857784177, "loss_nats": 1.2173852920532227, "step": 2670, "unclipped_grad_norm": 0.7113842368125916 }, { "bits_per_byte": 1.858153874543552, "learning_rate": 0.00024435100905948387, "loss_nats": 1.2879741191864014, "step": 2680, "unclipped_grad_norm": 0.7770810127258301 }, { "bits_per_byte": 1.784997614346039, "learning_rate": 0.00024389886365131793, "loss_nats": 1.2372660636901855, "step": 2690, "unclipped_grad_norm": 0.7249491810798645 }, { "bits_per_byte": 1.7001362149308035, "learning_rate": 0.00024344537007926163, "loss_nats": 1.1784446239471436, "step": 2700, "unclipped_grad_norm": 0.6978740096092224 }, { "bits_per_byte": 1.7671420315681605, "learning_rate": 0.00024299053609226878, "loss_nats": 1.2248895168304443, "step": 2710, "unclipped_grad_norm": 0.6759880185127258 }, { "bits_per_byte": 1.8404900524213854, "learning_rate": 0.00024253436946219733, "loss_nats": 1.2757304906845093, "step": 2720, "unclipped_grad_norm": 0.7600791454315186 }, { "bits_per_byte": 1.708491820040322, "learning_rate": 0.0002420768779836764, "loss_nats": 1.1842362880706787, "step": 2730, "unclipped_grad_norm": 0.6715761423110962 }, { "bits_per_byte": 1.7849604660934528, "learning_rate": 0.00024161806947397305, "loss_nats": 1.2372403144836426, "step": 2740, "unclipped_grad_norm": 0.7194302082061768 }, { "bits_per_byte": 1.7485555225241152, "learning_rate": 0.00024115795177285886, "loss_nats": 1.2120063304901123, "step": 2750, "unclipped_grad_norm": 0.6991336941719055 }, { "bits_per_byte": 1.8629487508496072, "learning_rate": 0.0002406965327424758, "loss_nats": 1.2912976741790771, "step": 2760, "unclipped_grad_norm": 0.8377771973609924 }, { "bits_per_byte": 1.7856594035865592, "learning_rate": 0.00024023382026720197, "loss_nats": 1.237724781036377, "step": 2770, "unclipped_grad_norm": 0.7150954604148865 }, { "bits_per_byte": 1.7901013714930438, "learning_rate": 0.00023976982225351698, "loss_nats": 1.2408037185668945, "step": 2780, "unclipped_grad_norm": 0.6968044638633728 }, { "bits_per_byte": 1.752669863480706, "learning_rate": 0.0002393045466298666, "loss_nats": 1.2148581743240356, "step": 2790, "unclipped_grad_norm": 0.7223197817802429 }, { "bits_per_byte": 1.7567675221201633, "learning_rate": 0.00023883800134652763, "loss_nats": 1.2176984548568726, "step": 2800, "unclipped_grad_norm": 0.7307369709014893 }, { "bits_per_byte": 1.7124591158304254, "learning_rate": 0.0002383701943754715, "loss_nats": 1.1869862079620361, "step": 2810, "unclipped_grad_norm": 0.7025272250175476 }, { "bits_per_byte": 1.7803284573577647, "learning_rate": 0.00023790113371022884, "loss_nats": 1.2340296506881714, "step": 2820, "unclipped_grad_norm": 0.7275472283363342 }, { "bits_per_byte": 1.7225665362216327, "learning_rate": 0.0002374308273657521, "loss_nats": 1.1939921379089355, "step": 2830, "unclipped_grad_norm": 0.6614095568656921 }, { "bits_per_byte": 1.7602923065495966, "learning_rate": 0.00023695928337827911, "loss_nats": 1.2201416492462158, "step": 2840, "unclipped_grad_norm": 0.7144313454627991 }, { "bits_per_byte": 1.7021394688480533, "learning_rate": 0.00023648650980519564, "loss_nats": 1.179833173751831, "step": 2850, "unclipped_grad_norm": 0.7220690846443176 }, { "bits_per_byte": 1.744293620453084, "learning_rate": 0.00023601251472489757, "loss_nats": 1.2090522050857544, "step": 2860, "unclipped_grad_norm": 0.7174125909805298 }, { "bits_per_byte": 1.7433115995166573, "learning_rate": 0.00023553730623665305, "loss_nats": 1.2083715200424194, "step": 2870, "unclipped_grad_norm": 0.730305552482605 }, { "bits_per_byte": 1.7421880368585705, "learning_rate": 0.00023506089246046403, "loss_nats": 1.2075927257537842, "step": 2880, "unclipped_grad_norm": 0.7075303792953491 }, { "bits_per_byte": 1.7085926018737276, "learning_rate": 0.00023458328153692736, "loss_nats": 1.1843061447143555, "step": 2890, "unclipped_grad_norm": 0.6744402050971985 }, { "bits_per_byte": 1.7378320602775172, "learning_rate": 0.0002341044816270961, "loss_nats": 1.204573392868042, "step": 2900, "unclipped_grad_norm": 0.6889722347259521 }, { "bits_per_byte": 1.7031551983840492, "learning_rate": 0.00023362450091233953, "loss_nats": 1.180537223815918, "step": 2910, "unclipped_grad_norm": 0.7552688717842102 }, { "bits_per_byte": 1.7508614659068837, "learning_rate": 0.00023314334759420388, "loss_nats": 1.2136046886444092, "step": 2920, "unclipped_grad_norm": 0.6863148808479309 }, { "bits_per_byte": 1.701510700276499, "learning_rate": 0.0002326610298942717, "loss_nats": 1.1793973445892334, "step": 2930, "unclipped_grad_norm": 0.6827563047409058 }, { "bits_per_byte": 1.7106488264474435, "learning_rate": 0.00023217755605402175, "loss_nats": 1.1857314109802246, "step": 2940, "unclipped_grad_norm": 0.6807736158370972 }, { "bits_per_byte": 1.7703209588867088, "learning_rate": 0.00023169293433468802, "loss_nats": 1.227092981338501, "step": 2950, "unclipped_grad_norm": 0.6866815686225891 }, { "bits_per_byte": 1.7651767858167513, "learning_rate": 0.00023120717301711857, "loss_nats": 1.2235273122787476, "step": 2960, "unclipped_grad_norm": 0.741482138633728 }, { "bits_per_byte": 1.7465264712092357, "learning_rate": 0.00023072028040163408, "loss_nats": 1.2105998992919922, "step": 2970, "unclipped_grad_norm": 0.6754798293113708 }, { "bits_per_byte": 1.7663952828981142, "learning_rate": 0.00023023226480788593, "loss_nats": 1.2243719100952148, "step": 2980, "unclipped_grad_norm": 0.688591480255127 }, { "bits_per_byte": 1.7069632382394535, "learning_rate": 0.00022974313457471416, "loss_nats": 1.1831767559051514, "step": 2990, "unclipped_grad_norm": 0.7298949956893921 }, { "bits_per_byte": 1.79113911480835, "learning_rate": 0.0002292528980600049, "loss_nats": 1.241523027420044, "step": 3000, "unclipped_grad_norm": 0.713433027267456 }, { "bits_per_byte": 1.6766162115641075, "learning_rate": 0.00022876156364054758, "loss_nats": 1.1621417999267578, "step": 3010, "unclipped_grad_norm": 0.7228698134422302 }, { "bits_per_byte": 1.7365501015979858, "learning_rate": 0.0002282691397118918, "loss_nats": 1.2036848068237305, "step": 3020, "unclipped_grad_norm": 0.6969108581542969 }, { "bits_per_byte": 1.6833617150962485, "learning_rate": 0.00022777563468820376, "loss_nats": 1.1668174266815186, "step": 3030, "unclipped_grad_norm": 0.669261634349823 }, { "bits_per_byte": 1.6671776317021056, "learning_rate": 0.00022728105700212282, "loss_nats": 1.1555994749069214, "step": 3040, "unclipped_grad_norm": 0.6839757561683655 }, { "bits_per_byte": 1.8241205717296125, "learning_rate": 0.00022678541510461698, "loss_nats": 1.2643840312957764, "step": 3050, "unclipped_grad_norm": 0.7963703870773315 }, { "bits_per_byte": 1.7604708245411922, "learning_rate": 0.00022628871746483876, "loss_nats": 1.2202653884887695, "step": 3060, "unclipped_grad_norm": 0.7280165553092957 }, { "bits_per_byte": 1.6594571585222233, "learning_rate": 0.0002257909725699804, "loss_nats": 1.1502480506896973, "step": 3070, "unclipped_grad_norm": 0.7014188766479492 }, { "bits_per_byte": 1.7706030104341237, "learning_rate": 0.00022529218892512886, "loss_nats": 1.2272884845733643, "step": 3080, "unclipped_grad_norm": 0.7385008335113525 }, { "bits_per_byte": 1.7493225651469633, "learning_rate": 0.00022479237505312034, "loss_nats": 1.2125380039215088, "step": 3090, "unclipped_grad_norm": 0.6976860165596008 }, { "bits_per_byte": 1.706454857523503, "learning_rate": 0.00022429153949439496, "loss_nats": 1.1828243732452393, "step": 3100, "unclipped_grad_norm": 0.7350029349327087 }, { "bits_per_byte": 1.8454660264587992, "learning_rate": 0.00022378969080685053, "loss_nats": 1.279179573059082, "step": 3110, "unclipped_grad_norm": 0.8262335658073425 }, { "bits_per_byte": 1.7237298268720687, "learning_rate": 0.0002232868375656965, "loss_nats": 1.194798469543457, "step": 3120, "unclipped_grad_norm": 0.7770306468009949 }, { "bits_per_byte": 1.6658043502349662, "learning_rate": 0.00022278298836330733, "loss_nats": 1.1546475887298584, "step": 3130, "unclipped_grad_norm": 0.6939164996147156 }, { "bits_per_byte": 1.7147224075157799, "learning_rate": 0.00022227815180907572, "loss_nats": 1.1885550022125244, "step": 3140, "unclipped_grad_norm": 0.744745135307312 }, { "bits_per_byte": 1.7012960659282221, "learning_rate": 0.00022177233652926546, "loss_nats": 1.179248571395874, "step": 3150, "unclipped_grad_norm": 0.6927894353866577 }, { "bits_per_byte": 1.700822425707746, "learning_rate": 0.00022126555116686416, "loss_nats": 1.1789202690124512, "step": 3160, "unclipped_grad_norm": 0.6962236166000366 }, { "bits_per_byte": 1.7730090477196945, "learning_rate": 0.00022075780438143532, "loss_nats": 1.2289562225341797, "step": 3170, "unclipped_grad_norm": 0.7581587433815002 }, { "bits_per_byte": 1.7445914944043783, "learning_rate": 0.00022024910484897065, "loss_nats": 1.2092586755752563, "step": 3180, "unclipped_grad_norm": 0.7753618359565735 }, { "bits_per_byte": 1.7320080397886992, "learning_rate": 0.00021973946126174158, "loss_nats": 1.2005364894866943, "step": 3190, "unclipped_grad_norm": 0.7117310166358948 }, { "bits_per_byte": 1.7215322325593436, "learning_rate": 0.00021922888232815086, "loss_nats": 1.1932752132415771, "step": 3200, "unclipped_grad_norm": 0.712298572063446 }, { "bits_per_byte": 1.6960516269728116, "learning_rate": 0.0002187173767725837, "loss_nats": 1.1756134033203125, "step": 3210, "unclipped_grad_norm": 0.7590813636779785 }, { "bits_per_byte": 1.6593746068498092, "learning_rate": 0.0002182049533352588, "loss_nats": 1.150190830230713, "step": 3220, "unclipped_grad_norm": 0.6987801194190979 }, { "bits_per_byte": 1.7324070395387008, "learning_rate": 0.00021769162077207884, "loss_nats": 1.2008130550384521, "step": 3230, "unclipped_grad_norm": 0.6938228607177734 }, { "bits_per_byte": 1.7474627447605329, "learning_rate": 0.00021717738785448102, "loss_nats": 1.2112488746643066, "step": 3240, "unclipped_grad_norm": 0.78226238489151 }, { "bits_per_byte": 1.7110863503112386, "learning_rate": 0.00021666226336928708, "loss_nats": 1.1860346794128418, "step": 3250, "unclipped_grad_norm": 0.6999595165252686 }, { "bits_per_byte": 1.7142417160066183, "learning_rate": 0.0002161462561185532, "loss_nats": 1.18822181224823, "step": 3260, "unclipped_grad_norm": 0.700433075428009 }, { "bits_per_byte": 1.748484321706658, "learning_rate": 0.00021562937491941957, "loss_nats": 1.2119569778442383, "step": 3270, "unclipped_grad_norm": 0.7274234294891357 }, { "bits_per_byte": 1.738793787261142, "learning_rate": 0.00021511162860395975, "loss_nats": 1.20524001121521, "step": 3280, "unclipped_grad_norm": 0.7183425426483154 }, { "bits_per_byte": 1.6962500949519075, "learning_rate": 0.0002145930260190298, "loss_nats": 1.175750970840454, "step": 3290, "unclipped_grad_norm": 0.7457373738288879 }, { "bits_per_byte": 1.7210296992535226, "learning_rate": 0.00021407357602611698, "loss_nats": 1.1929268836975098, "step": 3300, "unclipped_grad_norm": 0.7834483981132507 }, { "bits_per_byte": 1.6835460804979734, "learning_rate": 0.0002135532875011885, "loss_nats": 1.166945219039917, "step": 3310, "unclipped_grad_norm": 0.6523665189743042 }, { "bits_per_byte": 1.6871995079502522, "learning_rate": 0.0002130321693345397, "loss_nats": 1.1694775819778442, "step": 3320, "unclipped_grad_norm": 0.7348048686981201 }, { "bits_per_byte": 1.7008401399207849, "learning_rate": 0.00021251023043064224, "loss_nats": 1.178932547569275, "step": 3330, "unclipped_grad_norm": 0.7228167653083801 }, { "bits_per_byte": 1.643402750046831, "learning_rate": 0.00021198747970799188, "loss_nats": 1.1391199827194214, "step": 3340, "unclipped_grad_norm": 0.7094708681106567 }, { "bits_per_byte": 1.686364876145615, "learning_rate": 0.00021146392609895618, "loss_nats": 1.1688990592956543, "step": 3350, "unclipped_grad_norm": 0.7186631560325623 }, { "bits_per_byte": 1.7065993229502279, "learning_rate": 0.00021093957854962178, "loss_nats": 1.182924509048462, "step": 3360, "unclipped_grad_norm": 0.6950210332870483 }, { "bits_per_byte": 1.6548356407282379, "learning_rate": 0.00021041444601964157, "loss_nats": 1.1470446586608887, "step": 3370, "unclipped_grad_norm": 0.6926314830780029 }, { "bits_per_byte": 1.6926787032240904, "learning_rate": 0.00020988853748208164, "loss_nats": 1.1732754707336426, "step": 3380, "unclipped_grad_norm": 0.7107762694358826 }, { "bits_per_byte": 1.7003921251152874, "learning_rate": 0.00020936186192326782, "loss_nats": 1.1786220073699951, "step": 3390, "unclipped_grad_norm": 0.700124979019165 }, { "bits_per_byte": 1.6889117672222422, "learning_rate": 0.00020883442834263225, "loss_nats": 1.1706644296646118, "step": 3400, "unclipped_grad_norm": 0.7049819827079773 }, { "bits_per_byte": 1.6802605239358903, "learning_rate": 0.0002083062457525596, "loss_nats": 1.1646678447723389, "step": 3410, "unclipped_grad_norm": 0.6922183036804199 }, { "bits_per_byte": 1.664921907253389, "learning_rate": 0.00020777732317823306, "loss_nats": 1.1540359258651733, "step": 3420, "unclipped_grad_norm": 0.7144856452941895 }, { "bits_per_byte": 1.698013089105902, "learning_rate": 0.00020724766965748005, "loss_nats": 1.1769729852676392, "step": 3430, "unclipped_grad_norm": 0.6953245401382446 }, { "bits_per_byte": 1.7072490734051875, "learning_rate": 0.00020671729424061788, "loss_nats": 1.1833748817443848, "step": 3440, "unclipped_grad_norm": 0.727461040019989 }, { "bits_per_byte": 1.7747906159999822, "learning_rate": 0.00020618620599029906, "loss_nats": 1.2301911115646362, "step": 3450, "unclipped_grad_norm": 0.7409155368804932 }, { "bits_per_byte": 1.7588486841782542, "learning_rate": 0.00020565441398135649, "loss_nats": 1.2191410064697266, "step": 3460, "unclipped_grad_norm": 0.748551070690155 }, { "bits_per_byte": 1.7339454243506687, "learning_rate": 0.00020512192730064827, "loss_nats": 1.2018793821334839, "step": 3470, "unclipped_grad_norm": 0.7151429653167725 }, { "bits_per_byte": 1.7067943512763062, "learning_rate": 0.00020458875504690273, "loss_nats": 1.1830596923828125, "step": 3480, "unclipped_grad_norm": 0.7189637422561646 }, { "bits_per_byte": 1.6200879219826143, "learning_rate": 0.00020405490633056245, "loss_nats": 1.1229593753814697, "step": 3490, "unclipped_grad_norm": 0.6848708391189575 }, { "bits_per_byte": 1.6373195517031627, "learning_rate": 0.00020352039027362917, "loss_nats": 1.1349034309387207, "step": 3500, "unclipped_grad_norm": 0.6704034209251404 }, { "bits_per_byte": 1.6849599498707166, "learning_rate": 0.0002029852160095075, "loss_nats": 1.167925238609314, "step": 3510, "unclipped_grad_norm": 0.7942876219749451 }, { "bits_per_byte": 1.6514716600773613, "learning_rate": 0.000202449392682849, "loss_nats": 1.1447129249572754, "step": 3520, "unclipped_grad_norm": 0.7149106860160828 }, { "bits_per_byte": 1.6705435041621417, "learning_rate": 0.0002019129294493959, "loss_nats": 1.1579325199127197, "step": 3530, "unclipped_grad_norm": 0.7720932364463806 }, { "bits_per_byte": 1.6828739722984016, "learning_rate": 0.00020137583547582478, "loss_nats": 1.1664793491363525, "step": 3540, "unclipped_grad_norm": 0.6750161647796631 }, { "bits_per_byte": 1.7101762181228726, "learning_rate": 0.00020083811993958973, "loss_nats": 1.185403823852539, "step": 3550, "unclipped_grad_norm": 0.6993052363395691 }, { "bits_per_byte": 1.6591575647444203, "learning_rate": 0.00020029979202876563, "loss_nats": 1.1500403881072998, "step": 3560, "unclipped_grad_norm": 0.6886851787567139 }, { "bits_per_byte": 1.7182958630354075, "learning_rate": 0.00019976086094189118, "loss_nats": 1.1910319328308105, "step": 3570, "unclipped_grad_norm": 0.749891459941864 }, { "bits_per_byte": 1.7607525321233055, "learning_rate": 0.00019922133588781175, "loss_nats": 1.2204606533050537, "step": 3580, "unclipped_grad_norm": 0.8161401152610779 }, { "bits_per_byte": 1.690903842267186, "learning_rate": 0.0001986812260855219, "loss_nats": 1.1720452308654785, "step": 3590, "unclipped_grad_norm": 0.6925192475318909 }, { "bits_per_byte": 1.6772946831217612, "learning_rate": 0.00019814054076400787, "loss_nats": 1.1626120805740356, "step": 3600, "unclipped_grad_norm": 0.7061766982078552 }, { "bits_per_byte": 1.6673109182565242, "learning_rate": 0.00019759928916209004, "loss_nats": 1.1556918621063232, "step": 3610, "unclipped_grad_norm": 0.7162109017372131 }, { "bits_per_byte": 1.6598764522250269, "learning_rate": 0.00019705748052826493, "loss_nats": 1.150538682937622, "step": 3620, "unclipped_grad_norm": 0.6953511238098145 }, { "bits_per_byte": 1.6688818077895051, "learning_rate": 0.00019651512412054723, "loss_nats": 1.15678071975708, "step": 3630, "unclipped_grad_norm": 0.7265485525131226 }, { "bits_per_byte": 1.6378186453559664, "learning_rate": 0.00019597222920631144, "loss_nats": 1.135249376296997, "step": 3640, "unclipped_grad_norm": 0.68751460313797 }, { "bits_per_byte": 1.6996694540163617, "learning_rate": 0.0001954288050621338, "loss_nats": 1.1781210899353027, "step": 3650, "unclipped_grad_norm": 0.7145417928695679 }, { "bits_per_byte": 1.6073453834148872, "learning_rate": 0.00019488486097363363, "loss_nats": 1.1141269207000732, "step": 3660, "unclipped_grad_norm": 0.71860671043396 }, { "bits_per_byte": 1.62879970316207, "learning_rate": 0.00019434040623531452, "loss_nats": 1.1289979219436646, "step": 3670, "unclipped_grad_norm": 0.6769548654556274 }, { "bits_per_byte": 1.702227523965295, "learning_rate": 0.0001937954501504058, "loss_nats": 1.179894208908081, "step": 3680, "unclipped_grad_norm": 0.7466121315956116 }, { "bits_per_byte": 1.653425382991163, "learning_rate": 0.00019325000203070333, "loss_nats": 1.1460671424865723, "step": 3690, "unclipped_grad_norm": 0.6637522578239441 }, { "bits_per_byte": 1.5989233930021354, "learning_rate": 0.0001927040711964106, "loss_nats": 1.1082892417907715, "step": 3700, "unclipped_grad_norm": 0.7335078120231628 }, { "bits_per_byte": 1.6701862961962997, "learning_rate": 0.00019215766697597923, "loss_nats": 1.1576849222183228, "step": 3710, "unclipped_grad_norm": 0.7247957587242126 }, { "bits_per_byte": 1.6538059805975223, "learning_rate": 0.00019161079870594978, "loss_nats": 1.1463309526443481, "step": 3720, "unclipped_grad_norm": 0.7534144520759583 }, { "bits_per_byte": 1.7119505631318241, "learning_rate": 0.0001910634757307922, "loss_nats": 1.1866337060928345, "step": 3730, "unclipped_grad_norm": 0.6951875686645508 }, { "bits_per_byte": 1.6437530787066386, "learning_rate": 0.0001905157074027459, "loss_nats": 1.1393628120422363, "step": 3740, "unclipped_grad_norm": 0.7089563608169556 }, { "bits_per_byte": 1.6267918057132467, "learning_rate": 0.00018996750308166037, "loss_nats": 1.1276061534881592, "step": 3750, "unclipped_grad_norm": 0.6922760009765625 }, { "bits_per_byte": 1.6201740852906965, "learning_rate": 0.00018941887213483482, "loss_nats": 1.1230190992355347, "step": 3760, "unclipped_grad_norm": 0.6971311569213867 }, { "bits_per_byte": 1.6730486034546093, "learning_rate": 0.00018886982393685845, "loss_nats": 1.1596689224243164, "step": 3770, "unclipped_grad_norm": 0.6925700902938843 }, { "bits_per_byte": 1.6258194158052683, "learning_rate": 0.00018832036786945008, "loss_nats": 1.126932144165039, "step": 3780, "unclipped_grad_norm": 0.71840900182724 }, { "bits_per_byte": 1.6611288298886098, "learning_rate": 0.0001877705133212979, "loss_nats": 1.1514067649841309, "step": 3790, "unclipped_grad_norm": 0.6731990575790405 }, { "bits_per_byte": 1.622545554063444, "learning_rate": 0.00018722026968789906, "loss_nats": 1.1246628761291504, "step": 3800, "unclipped_grad_norm": 0.7068047523498535 }, { "bits_per_byte": 1.613386446009094, "learning_rate": 0.00018666964637139907, "loss_nats": 1.118314266204834, "step": 3810, "unclipped_grad_norm": 0.679490327835083 }, { "bits_per_byte": 1.723293162921528, "learning_rate": 0.00018611865278043115, "loss_nats": 1.1944957971572876, "step": 3820, "unclipped_grad_norm": 0.7167435884475708 }, { "bits_per_byte": 1.6410942269242994, "learning_rate": 0.00018556729832995555, "loss_nats": 1.1375198364257812, "step": 3830, "unclipped_grad_norm": 0.749890148639679 }, { "bits_per_byte": 1.6271811744348, "learning_rate": 0.00018501559244109864, "loss_nats": 1.1278760433197021, "step": 3840, "unclipped_grad_norm": 0.6869876980781555 }, { "bits_per_byte": 1.6520392028252087, "learning_rate": 0.00018446354454099178, "loss_nats": 1.145106315612793, "step": 3850, "unclipped_grad_norm": 0.6890377998352051 }, { "bits_per_byte": 1.6079827511189846, "learning_rate": 0.00018391116406261048, "loss_nats": 1.1145687103271484, "step": 3860, "unclipped_grad_norm": 0.717820942401886 }, { "bits_per_byte": 1.6751354409401786, "learning_rate": 0.00018335846044461305, "loss_nats": 1.1611154079437256, "step": 3870, "unclipped_grad_norm": 0.7368612289428711 }, { "bits_per_byte": 1.6338718155013159, "learning_rate": 0.0001828054431311793, "loss_nats": 1.1325136423110962, "step": 3880, "unclipped_grad_norm": 0.7661339640617371 }, { "bits_per_byte": 1.656585564200767, "learning_rate": 0.00018225212157184934, "loss_nats": 1.1482576131820679, "step": 3890, "unclipped_grad_norm": 0.7343658804893494 }, { "bits_per_byte": 1.6647341021986468, "learning_rate": 0.00018169850522136192, "loss_nats": 1.1539057493209839, "step": 3900, "unclipped_grad_norm": 0.7533658146858215 }, { "bits_per_byte": 1.6762937440937398, "learning_rate": 0.000181144603539493, "loss_nats": 1.16191828250885, "step": 3910, "unclipped_grad_norm": 0.7147406935691833 }, { "bits_per_byte": 1.6322013480134854, "learning_rate": 0.00018059042599089412, "loss_nats": 1.1313557624816895, "step": 3920, "unclipped_grad_norm": 0.6887187361717224 }, { "bits_per_byte": 1.6221845624792828, "learning_rate": 0.00018003598204493048, "loss_nats": 1.1244126558303833, "step": 3930, "unclipped_grad_norm": 0.7910217642784119 }, { "bits_per_byte": 1.63472175376188, "learning_rate": 0.00017948128117551936, "loss_nats": 1.1331027746200562, "step": 3940, "unclipped_grad_norm": 0.730076789855957 }, { "bits_per_byte": 1.6972579132859635, "learning_rate": 0.00017892633286096816, "loss_nats": 1.1764495372772217, "step": 3950, "unclipped_grad_norm": 0.7732967138290405 }, { "bits_per_byte": 1.7135035664691152, "learning_rate": 0.00017837114658381247, "loss_nats": 1.187710165977478, "step": 3960, "unclipped_grad_norm": 0.7405394911766052 }, { "bits_per_byte": 1.685746770498414, "learning_rate": 0.00017781573183065386, "loss_nats": 1.1684706211090088, "step": 3970, "unclipped_grad_norm": 0.7160924673080444 }, { "bits_per_byte": 1.6665311489175123, "learning_rate": 0.0001772600980919981, "loss_nats": 1.1551513671875, "step": 3980, "unclipped_grad_norm": 0.7374541759490967 }, { "bits_per_byte": 1.7040839046987084, "learning_rate": 0.0001767042548620927, "loss_nats": 1.1811809539794922, "step": 3990, "unclipped_grad_norm": 0.7841057181358337 }, { "bits_per_byte": 1.693502844087025, "learning_rate": 0.00017614821163876486, "loss_nats": 1.17384672164917, "step": 4000, "unclipped_grad_norm": 0.7609017491340637 }, { "bits_per_byte": 1.699465998540391, "learning_rate": 0.00017559197792325906, "loss_nats": 1.1779800653457642, "step": 4010, "unclipped_grad_norm": 0.7629905343055725 }, { "bits_per_byte": 1.5838037102341824, "learning_rate": 0.0001750355632200748, "loss_nats": 1.097809076309204, "step": 4020, "unclipped_grad_norm": 0.6868616342544556 }, { "bits_per_byte": 1.617556165379263, "learning_rate": 0.00017447897703680418, "loss_nats": 1.1212044954299927, "step": 4030, "unclipped_grad_norm": 0.749467670917511 }, { "bits_per_byte": 1.6189757101794846, "learning_rate": 0.0001739222288839694, "loss_nats": 1.1221884489059448, "step": 4040, "unclipped_grad_norm": 0.7323976755142212 }, { "bits_per_byte": 1.6111234982890412, "learning_rate": 0.00017336532827486026, "loss_nats": 1.1167457103729248, "step": 4050, "unclipped_grad_norm": 0.7304336428642273 }, { "bits_per_byte": 1.6428754512392856, "learning_rate": 0.00017280828472537163, "loss_nats": 1.1387544870376587, "step": 4060, "unclipped_grad_norm": 0.745829164981842 }, { "bits_per_byte": 1.6097484969953932, "learning_rate": 0.0001722511077538408, "loss_nats": 1.1157926321029663, "step": 4070, "unclipped_grad_norm": 0.7523763179779053 }, { "bits_per_byte": 1.6445053308215125, "learning_rate": 0.00017169380688088497, "loss_nats": 1.1398842334747314, "step": 4080, "unclipped_grad_norm": 0.7511109113693237 }, { "bits_per_byte": 1.6034616711931031, "learning_rate": 0.0001711363916292384, "loss_nats": 1.1114349365234375, "step": 4090, "unclipped_grad_norm": 0.7438745498657227 }, { "bits_per_byte": 1.561935084281071, "learning_rate": 0.00017057887152358974, "loss_nats": 1.082650899887085, "step": 4100, "unclipped_grad_norm": 0.6872318387031555 }, { "bits_per_byte": 1.6907129415247284, "learning_rate": 0.0001700212560904194, "loss_nats": 1.1719129085540771, "step": 4110, "unclipped_grad_norm": 0.7602059841156006 }, { "bits_per_byte": 1.6125045189754696, "learning_rate": 0.00016946355485783656, "loss_nats": 1.1177029609680176, "step": 4120, "unclipped_grad_norm": 0.7136654257774353 }, { "bits_per_byte": 1.5891117827704122, "learning_rate": 0.0001689057773554166, "loss_nats": 1.1014883518218994, "step": 4130, "unclipped_grad_norm": 0.7062129378318787 }, { "bits_per_byte": 1.6146775197691212, "learning_rate": 0.00016834793311403806, "loss_nats": 1.1192091703414917, "step": 4140, "unclipped_grad_norm": 0.7150430679321289 }, { "bits_per_byte": 1.6618133208390438, "learning_rate": 0.00016779003166571981, "loss_nats": 1.151881217956543, "step": 4150, "unclipped_grad_norm": 0.7448095679283142 }, { "bits_per_byte": 1.6228898633304711, "learning_rate": 0.00016723208254345838, "loss_nats": 1.124901533126831, "step": 4160, "unclipped_grad_norm": 0.7362097501754761 }, { "bits_per_byte": 1.5905947891688021, "learning_rate": 0.00016667409528106481, "loss_nats": 1.1025162935256958, "step": 4170, "unclipped_grad_norm": 0.7651577591896057 }, { "bits_per_byte": 1.5608787668394717, "learning_rate": 0.00016611607941300184, "loss_nats": 1.081918716430664, "step": 4180, "unclipped_grad_norm": 0.7135135531425476 }, { "bits_per_byte": 1.713072749928704, "learning_rate": 0.0001655580444742211, "loss_nats": 1.1874115467071533, "step": 4190, "unclipped_grad_norm": 0.7463747262954712 }, { "bits_per_byte": 1.5972510337051453, "learning_rate": 0.00016499999999999997, "loss_nats": 1.1071300506591797, "step": 4200, "unclipped_grad_norm": 0.7313898801803589 }, { "bits_per_byte": 1.6169509584308768, "learning_rate": 0.0001644419555257789, "loss_nats": 1.1207849979400635, "step": 4210, "unclipped_grad_norm": 0.7539675831794739 }, { "bits_per_byte": 1.6536405332873922, "learning_rate": 0.0001638839205869981, "loss_nats": 1.1462162733078003, "step": 4220, "unclipped_grad_norm": 0.7351350784301758 }, { "bits_per_byte": 1.6357479742395782, "learning_rate": 0.0001633259047189352, "loss_nats": 1.1338140964508057, "step": 4230, "unclipped_grad_norm": 0.7141795754432678 }, { "bits_per_byte": 1.6650933739562992, "learning_rate": 0.0001627679174565416, "loss_nats": 1.1541547775268555, "step": 4240, "unclipped_grad_norm": 0.7625938653945923 }, { "bits_per_byte": 1.6824343846427963, "learning_rate": 0.00016220996833428016, "loss_nats": 1.1661746501922607, "step": 4250, "unclipped_grad_norm": 0.702212393283844 }, { "bits_per_byte": 1.6672134040934852, "learning_rate": 0.00016165206688596191, "loss_nats": 1.155624270439148, "step": 4260, "unclipped_grad_norm": 0.775193989276886 }, { "bits_per_byte": 1.643573356836487, "learning_rate": 0.00016109422264458335, "loss_nats": 1.1392382383346558, "step": 4270, "unclipped_grad_norm": 0.7224892973899841 }, { "bits_per_byte": 1.6848497089915135, "learning_rate": 0.00016053644514216336, "loss_nats": 1.167848825454712, "step": 4280, "unclipped_grad_norm": 0.792568564414978 }, { "bits_per_byte": 1.6287462165576516, "learning_rate": 0.00015997874390958058, "loss_nats": 1.1289608478546143, "step": 4290, "unclipped_grad_norm": 0.7352984547615051 }, { "bits_per_byte": 1.5679441581022175, "learning_rate": 0.00015942112847641018, "loss_nats": 1.0868160724639893, "step": 4300, "unclipped_grad_norm": 0.7459802031517029 }, { "bits_per_byte": 1.7134996108681453, "learning_rate": 0.00015886360837076156, "loss_nats": 1.1877074241638184, "step": 4310, "unclipped_grad_norm": 0.8436940312385559 }, { "bits_per_byte": 1.585466610485375, "learning_rate": 0.00015830619311911495, "loss_nats": 1.0989617109298706, "step": 4320, "unclipped_grad_norm": 0.7326236367225647 }, { "bits_per_byte": 1.5870087789156617, "learning_rate": 0.00015774889224615915, "loss_nats": 1.1000306606292725, "step": 4330, "unclipped_grad_norm": 0.7154185175895691 }, { "bits_per_byte": 1.5845594020020737, "learning_rate": 0.00015719171527462834, "loss_nats": 1.0983328819274902, "step": 4340, "unclipped_grad_norm": 0.7299889326095581 }, { "bits_per_byte": 1.5087386145947432, "learning_rate": 0.0001566346717251397, "loss_nats": 1.0457779169082642, "step": 4350, "unclipped_grad_norm": 0.7116599082946777 }, { "bits_per_byte": 1.6642632137005844, "learning_rate": 0.00015607777111603056, "loss_nats": 1.1535793542861938, "step": 4360, "unclipped_grad_norm": 0.7841640114784241 }, { "bits_per_byte": 1.6118272512963718, "learning_rate": 0.00015552102296319576, "loss_nats": 1.1172335147857666, "step": 4370, "unclipped_grad_norm": 0.7682546973228455 }, { "bits_per_byte": 1.5975416843851036, "learning_rate": 0.00015496443677992516, "loss_nats": 1.1073315143585205, "step": 4380, "unclipped_grad_norm": 0.7542603611946106 }, { "bits_per_byte": 1.6190350441940322, "learning_rate": 0.0001544080220767409, "loss_nats": 1.1222295761108398, "step": 4390, "unclipped_grad_norm": 0.7354364991188049 }, { "bits_per_byte": 1.5791293937663822, "learning_rate": 0.00015385178836123511, "loss_nats": 1.0945690870285034, "step": 4400, "unclipped_grad_norm": 0.7447035908699036 }, { "bits_per_byte": 1.5777399459300616, "learning_rate": 0.00015329574513790726, "loss_nats": 1.0936059951782227, "step": 4410, "unclipped_grad_norm": 0.7433028221130371 }, { "bits_per_byte": 1.5774630538621723, "learning_rate": 0.00015273990190800186, "loss_nats": 1.093414068222046, "step": 4420, "unclipped_grad_norm": 0.7514091730117798 }, { "bits_per_byte": 1.5840658117940973, "learning_rate": 0.0001521842681693461, "loss_nats": 1.0979907512664795, "step": 4430, "unclipped_grad_norm": 0.748140811920166 }, { "bits_per_byte": 1.5992336497042918, "learning_rate": 0.00015162885341618752, "loss_nats": 1.108504295349121, "step": 4440, "unclipped_grad_norm": 0.7693572044372559 }, { "bits_per_byte": 1.5825853851354703, "learning_rate": 0.00015107366713903178, "loss_nats": 1.0969645977020264, "step": 4450, "unclipped_grad_norm": 0.7571823596954346 }, { "bits_per_byte": 1.5385392523882981, "learning_rate": 0.0001505187188244806, "loss_nats": 1.0664341449737549, "step": 4460, "unclipped_grad_norm": 0.7259396314620972 }, { "bits_per_byte": 1.6428487939284018, "learning_rate": 0.00014996401795506952, "loss_nats": 1.1387360095977783, "step": 4470, "unclipped_grad_norm": 0.7504101991653442 }, { "bits_per_byte": 1.5687149843433847, "learning_rate": 0.00014940957400910585, "loss_nats": 1.0873503684997559, "step": 4480, "unclipped_grad_norm": 0.7544133067131042 }, { "bits_per_byte": 1.5307804270772734, "learning_rate": 0.00014885539646050699, "loss_nats": 1.061056137084961, "step": 4490, "unclipped_grad_norm": 0.7395938634872437 }, { "bits_per_byte": 1.6123227333135077, "learning_rate": 0.00014830149477863805, "loss_nats": 1.1175769567489624, "step": 4500, "unclipped_grad_norm": 0.7624281644821167 }, { "bits_per_byte": 1.5895855949735391, "learning_rate": 0.00014774787842815066, "loss_nats": 1.1018167734146118, "step": 4510, "unclipped_grad_norm": 0.7442159652709961 }, { "bits_per_byte": 1.5720557473363947, "learning_rate": 0.00014719455686882066, "loss_nats": 1.0896660089492798, "step": 4520, "unclipped_grad_norm": 0.7299280166625977 }, { "bits_per_byte": 1.5930625682260326, "learning_rate": 0.00014664153955538695, "loss_nats": 1.10422682762146, "step": 4530, "unclipped_grad_norm": 0.7508688569068909 }, { "bits_per_byte": 1.551802554422838, "learning_rate": 0.00014608883593738946, "loss_nats": 1.0756275653839111, "step": 4540, "unclipped_grad_norm": 0.7326505780220032 }, { "bits_per_byte": 1.6245931795046165, "learning_rate": 0.00014553645545900817, "loss_nats": 1.126082181930542, "step": 4550, "unclipped_grad_norm": 0.7591264247894287 }, { "bits_per_byte": 1.6222865481912445, "learning_rate": 0.0001449844075589013, "loss_nats": 1.124483346939087, "step": 4560, "unclipped_grad_norm": 0.7490679621696472 }, { "bits_per_byte": 1.5636793323261218, "learning_rate": 0.0001444327016700444, "loss_nats": 1.083859920501709, "step": 4570, "unclipped_grad_norm": 0.7205959558486938 }, { "bits_per_byte": 1.5242402708302625, "learning_rate": 0.0001438813472195688, "loss_nats": 1.0565228462219238, "step": 4580, "unclipped_grad_norm": 0.7278034687042236 }, { "bits_per_byte": 1.649223158899981, "learning_rate": 0.00014333035362860093, "loss_nats": 1.1431543827056885, "step": 4590, "unclipped_grad_norm": 0.80039381980896 }, { "bits_per_byte": 1.621592598195013, "learning_rate": 0.00014277973031210088, "loss_nats": 1.1240023374557495, "step": 4600, "unclipped_grad_norm": 0.7594562768936157 }, { "bits_per_byte": 1.591440943811047, "learning_rate": 0.00014222948667870207, "loss_nats": 1.1031028032302856, "step": 4610, "unclipped_grad_norm": 0.7573962211608887 }, { "bits_per_byte": 1.5204752266375738, "learning_rate": 0.00014167963213054991, "loss_nats": 1.0539131164550781, "step": 4620, "unclipped_grad_norm": 0.7093440890312195 }, { "bits_per_byte": 1.6007937043302685, "learning_rate": 0.00014113017606314152, "loss_nats": 1.1095856428146362, "step": 4630, "unclipped_grad_norm": 0.7716385126113892 }, { "bits_per_byte": 1.566827990698118, "learning_rate": 0.00014058112786516518, "loss_nats": 1.0860424041748047, "step": 4640, "unclipped_grad_norm": 0.7477832436561584 }, { "bits_per_byte": 1.5540792607549598, "learning_rate": 0.0001400324969183396, "loss_nats": 1.0772056579589844, "step": 4650, "unclipped_grad_norm": 0.7563350200653076 }, { "bits_per_byte": 1.5711925664117143, "learning_rate": 0.00013948429259725406, "loss_nats": 1.0890676975250244, "step": 4660, "unclipped_grad_norm": 0.7477205991744995 }, { "bits_per_byte": 1.6065126434194095, "learning_rate": 0.00013893652426920776, "loss_nats": 1.1135497093200684, "step": 4670, "unclipped_grad_norm": 0.7434800267219543 }, { "bits_per_byte": 1.5613855997115644, "learning_rate": 0.00013838920129405016, "loss_nats": 1.0822700262069702, "step": 4680, "unclipped_grad_norm": 0.7484313249588013 }, { "bits_per_byte": 1.533432227570927, "learning_rate": 0.0001378423330240207, "loss_nats": 1.0628942251205444, "step": 4690, "unclipped_grad_norm": 0.7767173647880554 }, { "bits_per_byte": 1.6430371149310967, "learning_rate": 0.00013729592880358936, "loss_nats": 1.1388665437698364, "step": 4700, "unclipped_grad_norm": 0.7674224972724915 }, { "bits_per_byte": 1.552380416129737, "learning_rate": 0.00013674999796929659, "loss_nats": 1.0760281085968018, "step": 4710, "unclipped_grad_norm": 0.7836162447929382 }, { "bits_per_byte": 1.5203768525612804, "learning_rate": 0.00013620454984959417, "loss_nats": 1.053844928741455, "step": 4720, "unclipped_grad_norm": 0.7613595128059387 }, { "bits_per_byte": 1.6524877335786587, "learning_rate": 0.00013565959376468542, "loss_nats": 1.1454172134399414, "step": 4730, "unclipped_grad_norm": 0.7924392223358154 }, { "bits_per_byte": 1.587415861850254, "learning_rate": 0.00013511513902636634, "loss_nats": 1.1003128290176392, "step": 4740, "unclipped_grad_norm": 0.7647171020507812 }, { "bits_per_byte": 1.6396726183322676, "learning_rate": 0.0001345711949378662, "loss_nats": 1.1365344524383545, "step": 4750, "unclipped_grad_norm": 0.8181672096252441 }, { "bits_per_byte": 1.5091890371573529, "learning_rate": 0.00013402777079368853, "loss_nats": 1.0460901260375977, "step": 4760, "unclipped_grad_norm": 0.7407304048538208 }, { "bits_per_byte": 1.6098291568586478, "learning_rate": 0.0001334848758794528, "loss_nats": 1.1158485412597656, "step": 4770, "unclipped_grad_norm": 0.7601819038391113 }, { "bits_per_byte": 1.5582154435082103, "learning_rate": 0.00013294251947173504, "loss_nats": 1.0800726413726807, "step": 4780, "unclipped_grad_norm": 0.7307054996490479 }, { "bits_per_byte": 1.5944504682184955, "learning_rate": 0.00013240071083790996, "loss_nats": 1.1051888465881348, "step": 4790, "unclipped_grad_norm": 0.7617009282112122 }, { "bits_per_byte": 1.5823225956449518, "learning_rate": 0.0001318594592359921, "loss_nats": 1.0967824459075928, "step": 4800, "unclipped_grad_norm": 0.7428550124168396 }, { "bits_per_byte": 1.542078311377753, "learning_rate": 0.0001313187739144781, "loss_nats": 1.0688872337341309, "step": 4810, "unclipped_grad_norm": 0.773262083530426 }, { "bits_per_byte": 1.6108488419956133, "learning_rate": 0.0001307786641121882, "loss_nats": 1.1165553331375122, "step": 4820, "unclipped_grad_norm": 0.8227523565292358 }, { "bits_per_byte": 1.5982464692883394, "learning_rate": 0.00013023913905810876, "loss_nats": 1.1078200340270996, "step": 4830, "unclipped_grad_norm": 0.783271312713623 }, { "bits_per_byte": 1.6134836162068316, "learning_rate": 0.00012970020797123432, "loss_nats": 1.1183816194534302, "step": 4840, "unclipped_grad_norm": 0.7595719695091248 }, { "bits_per_byte": 1.5430993723759254, "learning_rate": 0.00012916188006041027, "loss_nats": 1.0695949792861938, "step": 4850, "unclipped_grad_norm": 0.8344528079032898 }, { "bits_per_byte": 1.551674427347945, "learning_rate": 0.00012862416452417516, "loss_nats": 1.0755387544631958, "step": 4860, "unclipped_grad_norm": 0.7924710512161255 }, { "bits_per_byte": 1.5551434893984988, "learning_rate": 0.00012808707055060405, "loss_nats": 1.0779433250427246, "step": 4870, "unclipped_grad_norm": 0.7731344103813171 }, { "bits_per_byte": 1.7162552888829201, "learning_rate": 0.000127550607317151, "loss_nats": 1.1896175146102905, "step": 4880, "unclipped_grad_norm": 0.8454892039299011 }, { "bits_per_byte": 1.5015098398136777, "learning_rate": 0.00012701478399049248, "loss_nats": 1.0407673120498657, "step": 4890, "unclipped_grad_norm": 0.7279435396194458 }, { "bits_per_byte": 1.6017640304464364, "learning_rate": 0.0001264796097263708, "loss_nats": 1.1102582216262817, "step": 4900, "unclipped_grad_norm": 0.7495299577713013 }, { "bits_per_byte": 1.5795956387328711, "learning_rate": 0.0001259450936694375, "loss_nats": 1.0948922634124756, "step": 4910, "unclipped_grad_norm": 0.8354763388633728 }, { "bits_per_byte": 1.5130457481029513, "learning_rate": 0.00012541124495309727, "loss_nats": 1.048763394355774, "step": 4920, "unclipped_grad_norm": 0.7128972411155701 }, { "bits_per_byte": 1.5263470583033318, "learning_rate": 0.00012487807269935167, "loss_nats": 1.057983160018921, "step": 4930, "unclipped_grad_norm": 0.7301722168922424 }, { "bits_per_byte": 1.56106777577277, "learning_rate": 0.00012434558601864349, "loss_nats": 1.0820497274398804, "step": 4940, "unclipped_grad_norm": 0.7844029664993286 }, { "bits_per_byte": 1.5809642766684375, "learning_rate": 0.0001238137940097009, "loss_nats": 1.0958409309387207, "step": 4950, "unclipped_grad_norm": 0.7815173864364624 }, { "bits_per_byte": 1.6004612618661507, "learning_rate": 0.0001232827057593821, "loss_nats": 1.1093552112579346, "step": 4960, "unclipped_grad_norm": 0.8055645227432251 }, { "bits_per_byte": 1.5618184800437862, "learning_rate": 0.00012275233034251992, "loss_nats": 1.0825700759887695, "step": 4970, "unclipped_grad_norm": 0.7773133516311646 }, { "bits_per_byte": 1.606830467358204, "learning_rate": 0.0001222226768217669, "loss_nats": 1.1137700080871582, "step": 4980, "unclipped_grad_norm": 0.7920810580253601 }, { "bits_per_byte": 1.567882244347907, "learning_rate": 0.00012169375424744034, "loss_nats": 1.086773157119751, "step": 4990, "unclipped_grad_norm": 0.7929123640060425 }, { "bits_per_byte": 1.5708692390280923, "learning_rate": 0.00012116557165736772, "loss_nats": 1.088843584060669, "step": 5000, "unclipped_grad_norm": 0.762309730052948 }, { "bits_per_byte": 1.6157285057485438, "learning_rate": 0.00012063813807673214, "loss_nats": 1.1199376583099365, "step": 5010, "unclipped_grad_norm": 0.7734562158584595 }, { "bits_per_byte": 1.5685849654593325, "learning_rate": 0.00012011146251791832, "loss_nats": 1.0872602462768555, "step": 5020, "unclipped_grad_norm": 0.8221466541290283 }, { "bits_per_byte": 1.5125002191344146, "learning_rate": 0.0001195855539803584, "loss_nats": 1.0483852624893188, "step": 5030, "unclipped_grad_norm": 0.7515868544578552 }, { "bits_per_byte": 1.5645020973278494, "learning_rate": 0.00011906042145037817, "loss_nats": 1.08443021774292, "step": 5040, "unclipped_grad_norm": 0.7403401732444763 }, { "bits_per_byte": 1.5465288784167808, "learning_rate": 0.0001185360739010438, "loss_nats": 1.071972131729126, "step": 5050, "unclipped_grad_norm": 0.8082988262176514 }, { "bits_per_byte": 1.6139199361920706, "learning_rate": 0.00011801252029200808, "loss_nats": 1.1186840534210205, "step": 5060, "unclipped_grad_norm": 0.8066977262496948 }, { "bits_per_byte": 1.5533101543403012, "learning_rate": 0.00011748976956935777, "loss_nats": 1.0766725540161133, "step": 5070, "unclipped_grad_norm": 0.7855141758918762 }, { "bits_per_byte": 1.6163216739113697, "learning_rate": 0.00011696783066546024, "loss_nats": 1.1203488111495972, "step": 5080, "unclipped_grad_norm": 0.776370644569397 }, { "bits_per_byte": 1.5614495772576855, "learning_rate": 0.00011644671249881147, "loss_nats": 1.082314372062683, "step": 5090, "unclipped_grad_norm": 0.776827871799469 }, { "bits_per_byte": 1.5419546558517827, "learning_rate": 0.00011592642397388293, "loss_nats": 1.0688015222549438, "step": 5100, "unclipped_grad_norm": 0.7785685062408447 }, { "bits_per_byte": 1.5800214677764075, "learning_rate": 0.00011540697398097015, "loss_nats": 1.0951874256134033, "step": 5110, "unclipped_grad_norm": 0.8104537725448608 }, { "bits_per_byte": 1.487382496941203, "learning_rate": 0.00011488837139604016, "loss_nats": 1.0309749841690063, "step": 5120, "unclipped_grad_norm": 0.7389267086982727 }, { "bits_per_byte": 1.6243671943013827, "learning_rate": 0.00011437062508058039, "loss_nats": 1.1259255409240723, "step": 5130, "unclipped_grad_norm": 0.8053697943687439 }, { "bits_per_byte": 1.5876562935961602, "learning_rate": 0.00011385374388144673, "loss_nats": 1.1004794836044312, "step": 5140, "unclipped_grad_norm": 0.8099423050880432 }, { "bits_per_byte": 1.5188835272038386, "learning_rate": 0.00011333773663071286, "loss_nats": 1.0528098344802856, "step": 5150, "unclipped_grad_norm": 0.7463088035583496 }, { "bits_per_byte": 1.5574305146896725, "learning_rate": 0.00011282261214551897, "loss_nats": 1.079528570175171, "step": 5160, "unclipped_grad_norm": 0.7795574069023132 }, { "bits_per_byte": 1.604379370618107, "learning_rate": 0.00011230837922792112, "loss_nats": 1.1120710372924805, "step": 5170, "unclipped_grad_norm": 0.8112488985061646 }, { "bits_per_byte": 1.570943535533265, "learning_rate": 0.0001117950466647412, "loss_nats": 1.0888950824737549, "step": 5180, "unclipped_grad_norm": 0.7960402965545654 }, { "bits_per_byte": 1.4686855750353036, "learning_rate": 0.00011128262322741627, "loss_nats": 1.0180152654647827, "step": 5190, "unclipped_grad_norm": 0.7735135555267334 }, { "bits_per_byte": 1.5401951013508053, "learning_rate": 0.00011077111767184913, "loss_nats": 1.0675818920135498, "step": 5200, "unclipped_grad_norm": 0.7935121655464172 }, { "bits_per_byte": 1.522642552003747, "learning_rate": 0.00011026053873825837, "loss_nats": 1.055415391921997, "step": 5210, "unclipped_grad_norm": 0.7616920471191406 }, { "bits_per_byte": 1.5296819738862126, "learning_rate": 0.0001097508951510293, "loss_nats": 1.0602947473526, "step": 5220, "unclipped_grad_norm": 0.799717128276825 }, { "bits_per_byte": 1.60920967535024, "learning_rate": 0.0001092421956185646, "loss_nats": 1.1154191493988037, "step": 5230, "unclipped_grad_norm": 0.8263183832168579 }, { "bits_per_byte": 1.5759482306733728, "learning_rate": 0.00010873444883313581, "loss_nats": 1.0923640727996826, "step": 5240, "unclipped_grad_norm": 0.9089767336845398 }, { "bits_per_byte": 1.5751560785834986, "learning_rate": 0.00010822766347073447, "loss_nats": 1.0918149948120117, "step": 5250, "unclipped_grad_norm": 0.8223497271537781 }, { "bits_per_byte": 1.518346597367845, "learning_rate": 0.00010772184819092424, "loss_nats": 1.052437663078308, "step": 5260, "unclipped_grad_norm": 0.7989961504936218 }, { "bits_per_byte": 1.5652323356634128, "learning_rate": 0.0001072170116366926, "loss_nats": 1.0849363803863525, "step": 5270, "unclipped_grad_norm": 0.7858845591545105 }, { "bits_per_byte": 1.4824085866955998, "learning_rate": 0.00010671316243430345, "loss_nats": 1.0275273323059082, "step": 5280, "unclipped_grad_norm": 0.7963522672653198 }, { "bits_per_byte": 1.5109090356486319, "learning_rate": 0.00010621030919314944, "loss_nats": 1.047282338142395, "step": 5290, "unclipped_grad_norm": 0.7786324620246887 }, { "bits_per_byte": 1.4973765807654917, "learning_rate": 0.000105708460505605, "loss_nats": 1.0379023551940918, "step": 5300, "unclipped_grad_norm": 0.8200122714042664 }, { "bits_per_byte": 1.4979238295605373, "learning_rate": 0.00010520762494687964, "loss_nats": 1.0382816791534424, "step": 5310, "unclipped_grad_norm": 0.798004686832428 }, { "bits_per_byte": 1.4963071926424267, "learning_rate": 0.00010470781107487111, "loss_nats": 1.037161111831665, "step": 5320, "unclipped_grad_norm": 0.7881578207015991 }, { "bits_per_byte": 1.5549873291515153, "learning_rate": 0.00010420902743001955, "loss_nats": 1.0778350830078125, "step": 5330, "unclipped_grad_norm": 0.8092938661575317 }, { "bits_per_byte": 1.6144941862633015, "learning_rate": 0.00010371128253516117, "loss_nats": 1.1190820932388306, "step": 5340, "unclipped_grad_norm": 0.8349001407623291 }, { "bits_per_byte": 1.4860422361430292, "learning_rate": 0.000103214584895383, "loss_nats": 1.030045986175537, "step": 5350, "unclipped_grad_norm": 0.7843849062919617 }, { "bits_per_byte": 1.5082066722556244, "learning_rate": 0.00010271894299787715, "loss_nats": 1.0454092025756836, "step": 5360, "unclipped_grad_norm": 0.7918040752410889 }, { "bits_per_byte": 1.577472340925319, "learning_rate": 0.00010222436531179622, "loss_nats": 1.0934205055236816, "step": 5370, "unclipped_grad_norm": 0.8359649181365967 }, { "bits_per_byte": 1.5013394050066726, "learning_rate": 0.0001017308602881082, "loss_nats": 1.040649175643921, "step": 5380, "unclipped_grad_norm": 0.7734270095825195 }, { "bits_per_byte": 1.5016145772480531, "learning_rate": 0.00010123843635945242, "loss_nats": 1.0408399105072021, "step": 5390, "unclipped_grad_norm": 0.7885358333587646 }, { "bits_per_byte": 1.5491196250693782, "learning_rate": 0.00010074710193999505, "loss_nats": 1.073767900466919, "step": 5400, "unclipped_grad_norm": 0.8202832341194153 }, { "bits_per_byte": 1.4666546319112648, "learning_rate": 0.0001002568654252858, "loss_nats": 1.0166075229644775, "step": 5410, "unclipped_grad_norm": 0.8083077073097229 }, { "bits_per_byte": 1.6009971598062391, "learning_rate": 9.976773519211404e-05, "loss_nats": 1.1097266674041748, "step": 5420, "unclipped_grad_norm": 0.847724974155426 }, { "bits_per_byte": 1.5289176829857782, "learning_rate": 9.92797195983659e-05, "loss_nats": 1.0597649812698364, "step": 5430, "unclipped_grad_norm": 0.8225540518760681 }, { "bits_per_byte": 1.530357349756151, "learning_rate": 9.879282698288142e-05, "loss_nats": 1.060762882232666, "step": 5440, "unclipped_grad_norm": 0.8045809864997864 }, { "bits_per_byte": 1.5424515137301253, "learning_rate": 9.830706566531197e-05, "loss_nats": 1.069145917892456, "step": 5450, "unclipped_grad_norm": 0.8051854968070984 }, { "bits_per_byte": 1.5648894022575925, "learning_rate": 9.782244394597826e-05, "loss_nats": 1.0846986770629883, "step": 5460, "unclipped_grad_norm": 0.7835660576820374 }, { "bits_per_byte": 1.5450825043230245, "learning_rate": 9.733897010572829e-05, "loss_nats": 1.070969581604004, "step": 5470, "unclipped_grad_norm": 0.8599129915237427 }, { "bits_per_byte": 1.4780804713039863, "learning_rate": 9.685665240579612e-05, "loss_nats": 1.0245273113250732, "step": 5480, "unclipped_grad_norm": 0.8000594973564148 }, { "bits_per_byte": 1.5424891779306642, "learning_rate": 9.637549908766041e-05, "loss_nats": 1.0691720247268677, "step": 5490, "unclipped_grad_norm": 0.8055532574653625 }, { "bits_per_byte": 1.5569413960306187, "learning_rate": 9.589551837290389e-05, "loss_nats": 1.0791895389556885, "step": 5500, "unclipped_grad_norm": 0.8300883769989014 }, { "bits_per_byte": 1.5474785666148452, "learning_rate": 9.541671846307257e-05, "loss_nats": 1.0726304054260254, "step": 5510, "unclipped_grad_norm": 0.826352059841156 }, { "bits_per_byte": 1.5350409532870977, "learning_rate": 9.493910753953597e-05, "loss_nats": 1.0640093088150024, "step": 5520, "unclipped_grad_norm": 0.853265643119812 }, { "bits_per_byte": 1.5635150888945477, "learning_rate": 9.446269376334689e-05, "loss_nats": 1.083746075630188, "step": 5530, "unclipped_grad_norm": 0.8060055375099182 }, { "bits_per_byte": 1.5843347926600468, "learning_rate": 9.39874852751024e-05, "loss_nats": 1.098177194595337, "step": 5540, "unclipped_grad_norm": 0.8706573843955994 }, { "bits_per_byte": 1.5165710484803372, "learning_rate": 9.351349019480433e-05, "loss_nats": 1.0512069463729858, "step": 5550, "unclipped_grad_norm": 0.7865509986877441 }, { "bits_per_byte": 1.5106121935932426, "learning_rate": 9.304071662172086e-05, "loss_nats": 1.0470765829086304, "step": 5560, "unclipped_grad_norm": 0.8064424991607666 }, { "bits_per_byte": 1.5654490338035, "learning_rate": 9.256917263424787e-05, "loss_nats": 1.0850865840911865, "step": 5570, "unclipped_grad_norm": 0.8376733064651489 }, { "bits_per_byte": 1.4567845475782482, "learning_rate": 9.209886628977114e-05, "loss_nats": 1.0097661018371582, "step": 5580, "unclipped_grad_norm": 0.7972759008407593 }, { "bits_per_byte": 1.5257076268074239, "learning_rate": 9.162980562452843e-05, "loss_nats": 1.057539939880371, "step": 5590, "unclipped_grad_norm": 0.8159511089324951 }, { "bits_per_byte": 1.5137768463517691, "learning_rate": 9.116199865347238e-05, "loss_nats": 1.0492701530456543, "step": 5600, "unclipped_grad_norm": 0.7912799715995789 }, { "bits_per_byte": 1.5312812405565859, "learning_rate": 9.069545337013333e-05, "loss_nats": 1.0614032745361328, "step": 5610, "unclipped_grad_norm": 0.7786760330200195 }, { "bits_per_byte": 1.4626550033827992, "learning_rate": 9.0230177746483e-05, "loss_nats": 1.0138351917266846, "step": 5620, "unclipped_grad_norm": 0.8055174350738525 }, { "bits_per_byte": 1.4690646247978054, "learning_rate": 8.976617973279797e-05, "loss_nats": 1.0182780027389526, "step": 5630, "unclipped_grad_norm": 0.7851108312606812 }, { "bits_per_byte": 1.4663743001903584, "learning_rate": 8.930346725752419e-05, "loss_nats": 1.0164132118225098, "step": 5640, "unclipped_grad_norm": 0.8267287015914917 }, { "bits_per_byte": 1.5738650048234715, "learning_rate": 8.88420482271411e-05, "loss_nats": 1.090920090675354, "step": 5650, "unclipped_grad_norm": 0.8359448313713074 }, { "bits_per_byte": 1.5300369460775933, "learning_rate": 8.838193052602691e-05, "loss_nats": 1.060540795326233, "step": 5660, "unclipped_grad_norm": 0.8635727167129517 }, { "bits_per_byte": 1.5230305448640935, "learning_rate": 8.792312201632352e-05, "loss_nats": 1.0556843280792236, "step": 5670, "unclipped_grad_norm": 0.8095638751983643 }, { "bits_per_byte": 1.524155483383387, "learning_rate": 8.746563053780261e-05, "loss_nats": 1.0564640760421753, "step": 5680, "unclipped_grad_norm": 0.8353701233863831 }, { "bits_per_byte": 1.566590998605229, "learning_rate": 8.700946390773114e-05, "loss_nats": 1.0858781337738037, "step": 5690, "unclipped_grad_norm": 0.8297926187515259 }, { "bits_per_byte": 1.499949785187701, "learning_rate": 8.655462992073833e-05, "loss_nats": 1.0396859645843506, "step": 5700, "unclipped_grad_norm": 0.8335801959037781 }, { "bits_per_byte": 1.5306388853556132, "learning_rate": 8.610113634868199e-05, "loss_nats": 1.0609580278396606, "step": 5710, "unclipped_grad_norm": 0.8218200206756592 }, { "bits_per_byte": 1.5663932185567369, "learning_rate": 8.564899094051613e-05, "loss_nats": 1.0857410430908203, "step": 5720, "unclipped_grad_norm": 0.8909743428230286 }, { "bits_per_byte": 1.4614645394735268, "learning_rate": 8.519820142215816e-05, "loss_nats": 1.013010025024414, "step": 5730, "unclipped_grad_norm": 0.7809724807739258 }, { "bits_per_byte": 1.5314872757723195, "learning_rate": 8.474877549635728e-05, "loss_nats": 1.0615460872650146, "step": 5740, "unclipped_grad_norm": 0.8163876533508301 }, { "bits_per_byte": 1.499649159513993, "learning_rate": 8.430072084256247e-05, "loss_nats": 1.0394775867462158, "step": 5750, "unclipped_grad_norm": 0.8667106628417969 }, { "bits_per_byte": 1.4999243317553734, "learning_rate": 8.385404511679161e-05, "loss_nats": 1.039668321609497, "step": 5760, "unclipped_grad_norm": 0.8516601324081421 }, { "bits_per_byte": 1.450708744488567, "learning_rate": 8.340875595150034e-05, "loss_nats": 1.0055546760559082, "step": 5770, "unclipped_grad_norm": 0.7943668365478516 }, { "bits_per_byte": 1.5466515020468459, "learning_rate": 8.296486095545203e-05, "loss_nats": 1.0720571279525757, "step": 5780, "unclipped_grad_norm": 0.8325088024139404 }, { "bits_per_byte": 1.494320105094358, "learning_rate": 8.252236771358729e-05, "loss_nats": 1.0357837677001953, "step": 5790, "unclipped_grad_norm": 0.8366720080375671 }, { "bits_per_byte": 1.4906410522271007, "learning_rate": 8.208128378689484e-05, "loss_nats": 1.033233642578125, "step": 5800, "unclipped_grad_norm": 0.8333538770675659 }, { "bits_per_byte": 1.5321304908865463, "learning_rate": 8.16416167122819e-05, "loss_nats": 1.0619919300079346, "step": 5810, "unclipped_grad_norm": 0.8446563482284546 }, { "bits_per_byte": 1.51830377368778, "learning_rate": 8.120337400244578e-05, "loss_nats": 1.05240797996521, "step": 5820, "unclipped_grad_norm": 0.8496891856193542 }, { "bits_per_byte": 1.4932847695361637, "learning_rate": 8.076656314574521e-05, "loss_nats": 1.0350661277770996, "step": 5830, "unclipped_grad_norm": 0.8205961585044861 }, { "bits_per_byte": 1.456290785387621, "learning_rate": 8.03311916060725e-05, "loss_nats": 1.009423851966858, "step": 5840, "unclipped_grad_norm": 0.8371089100837708 }, { "bits_per_byte": 1.454681199758196, "learning_rate": 7.989726682272606e-05, "loss_nats": 1.0083081722259521, "step": 5850, "unclipped_grad_norm": 0.8174164295196533 }, { "bits_per_byte": 1.4966561454410274, "learning_rate": 7.946479621028318e-05, "loss_nats": 1.0374029874801636, "step": 5860, "unclipped_grad_norm": 0.9058852791786194 }, { "bits_per_byte": 1.5418609253070623, "learning_rate": 7.903378715847343e-05, "loss_nats": 1.0687365531921387, "step": 5870, "unclipped_grad_norm": 0.8718268275260925 }, { "bits_per_byte": 1.5420466665699941, "learning_rate": 7.860424703205224e-05, "loss_nats": 1.0688652992248535, "step": 5880, "unclipped_grad_norm": 0.8315351009368896 }, { "bits_per_byte": 1.525186175410008, "learning_rate": 7.817618317067534e-05, "loss_nats": 1.0571784973144531, "step": 5890, "unclipped_grad_norm": 0.8407886028289795 }, { "bits_per_byte": 1.4989159974733648, "learning_rate": 7.774960288877295e-05, "loss_nats": 1.0389693975448608, "step": 5900, "unclipped_grad_norm": 0.8770707249641418 }, { "bits_per_byte": 1.4937374278732347, "learning_rate": 7.732451347542522e-05, "loss_nats": 1.0353798866271973, "step": 5910, "unclipped_grad_norm": 0.8901946544647217 }, { "bits_per_byte": 1.4996551789067731, "learning_rate": 7.690092219423722e-05, "loss_nats": 1.03948175907135, "step": 5920, "unclipped_grad_norm": 0.85027015209198 }, { "bits_per_byte": 1.5573538104273876, "learning_rate": 7.647883628321536e-05, "loss_nats": 1.0794754028320312, "step": 5930, "unclipped_grad_norm": 0.8851437568664551 }, { "bits_per_byte": 1.5404470559343193, "learning_rate": 7.605826295464316e-05, "loss_nats": 1.0677565336227417, "step": 5940, "unclipped_grad_norm": 0.8759228587150574 }, { "bits_per_byte": 1.5066732749405318, "learning_rate": 7.563920939495851e-05, "loss_nats": 1.0443463325500488, "step": 5950, "unclipped_grad_norm": 0.8818607926368713 }, { "bits_per_byte": 1.5044743047665998, "learning_rate": 7.522168276463047e-05, "loss_nats": 1.0428221225738525, "step": 5960, "unclipped_grad_norm": 0.816109299659729 }, { "bits_per_byte": 1.5173669841885302, "learning_rate": 7.480569019803719e-05, "loss_nats": 1.0517586469650269, "step": 5970, "unclipped_grad_norm": 0.8590478301048279 }, { "bits_per_byte": 1.4670959393933787, "learning_rate": 7.439123880334388e-05, "loss_nats": 1.0169134140014648, "step": 5980, "unclipped_grad_norm": 0.8544116616249084 }, { "bits_per_byte": 1.4408131207318973, "learning_rate": 7.39783356623813e-05, "loss_nats": 0.9986955523490906, "step": 5990, "unclipped_grad_norm": 0.8525132536888123 }, { "bits_per_byte": 1.5172739415744134, "learning_rate": 7.356698783052497e-05, "loss_nats": 1.0516941547393799, "step": 6000, "unclipped_grad_norm": 0.860197126865387 }, { "bits_per_byte": 1.499894750739425, "learning_rate": 7.315720233657431e-05, "loss_nats": 1.0396478176116943, "step": 6010, "unclipped_grad_norm": 0.8407278060913086 }, { "bits_per_byte": 1.4776142263374972, "learning_rate": 7.274898618263286e-05, "loss_nats": 1.024204134941101, "step": 6020, "unclipped_grad_norm": 0.8550556302070618 }, { "bits_per_byte": 1.5367169242197558, "learning_rate": 7.234234634398831e-05, "loss_nats": 1.0651710033416748, "step": 6030, "unclipped_grad_norm": 0.8657232522964478 }, { "bits_per_byte": 1.4931389282482321, "learning_rate": 7.193728976899362e-05, "loss_nats": 1.0349650382995605, "step": 6040, "unclipped_grad_norm": 0.8972119092941284 }, { "bits_per_byte": 1.498401597364634, "learning_rate": 7.153382337894797e-05, "loss_nats": 1.0386128425598145, "step": 6050, "unclipped_grad_norm": 0.8974336981773376 }, { "bits_per_byte": 1.5272425719663745, "learning_rate": 7.113195406797887e-05, "loss_nats": 1.0586038827896118, "step": 6060, "unclipped_grad_norm": 0.8948551416397095 }, { "bits_per_byte": 1.5059438965182226, "learning_rate": 7.073168870292397e-05, "loss_nats": 1.043840765953064, "step": 6070, "unclipped_grad_norm": 0.843971312046051 }, { "bits_per_byte": 1.4791374766761891, "learning_rate": 7.033303412321404e-05, "loss_nats": 1.0252599716186523, "step": 6080, "unclipped_grad_norm": 0.8345775008201599 }, { "bits_per_byte": 1.4884991802932552, "learning_rate": 6.993599714075586e-05, "loss_nats": 1.0317490100860596, "step": 6090, "unclipped_grad_norm": 0.8372970223426819 }, { "bits_per_byte": 1.486896301987214, "learning_rate": 6.954058453981609e-05, "loss_nats": 1.0306379795074463, "step": 6100, "unclipped_grad_norm": 0.827723503112793 }, { "bits_per_byte": 1.4668594632484424, "learning_rate": 6.914680307690504e-05, "loss_nats": 1.0167495012283325, "step": 6110, "unclipped_grad_norm": 0.871588945388794 }, { "bits_per_byte": 1.4677823221529722, "learning_rate": 6.875465948066139e-05, "loss_nats": 1.017389178276062, "step": 6120, "unclipped_grad_norm": 0.8457695245742798 }, { "bits_per_byte": 1.5061057321926845, "learning_rate": 6.836416045173733e-05, "loss_nats": 1.0439529418945312, "step": 6130, "unclipped_grad_norm": 0.9012824296951294 }, { "bits_per_byte": 1.5178127632195666, "learning_rate": 6.797531266268377e-05, "loss_nats": 1.0520676374435425, "step": 6140, "unclipped_grad_norm": 0.8450613617897034 }, { "bits_per_byte": 1.4675592606548031, "learning_rate": 6.758812275783663e-05, "loss_nats": 1.0172345638275146, "step": 6150, "unclipped_grad_norm": 0.885280191898346 }, { "bits_per_byte": 1.5554258849112155, "learning_rate": 6.720259735320298e-05, "loss_nats": 1.078139066696167, "step": 6160, "unclipped_grad_norm": 0.9099593758583069 }, { "bits_per_byte": 1.421598961003528, "learning_rate": 6.681874303634842e-05, "loss_nats": 0.985377311706543, "step": 6170, "unclipped_grad_norm": 0.7732160091400146 }, { "bits_per_byte": 1.5028951600663776, "learning_rate": 6.643656636628402e-05, "loss_nats": 1.0417275428771973, "step": 6180, "unclipped_grad_norm": 0.869156002998352 }, { "bits_per_byte": 1.4956368042693635, "learning_rate": 6.605607387335466e-05, "loss_nats": 1.036696434020996, "step": 6190, "unclipped_grad_norm": 0.8908746838569641 }, { "bits_per_byte": 1.4533810109176732, "learning_rate": 6.567727205912724e-05, "loss_nats": 1.0074069499969482, "step": 6200, "unclipped_grad_norm": 0.8600793480873108 }, { "bits_per_byte": 1.475978499345141, "learning_rate": 6.530016739627959e-05, "loss_nats": 1.0230703353881836, "step": 6210, "unclipped_grad_norm": 0.8739057779312134 }, { "bits_per_byte": 1.4582183669384914, "learning_rate": 6.492476632848993e-05, "loss_nats": 1.010759949684143, "step": 6220, "unclipped_grad_norm": 0.8542900681495667 }, { "bits_per_byte": 1.4401650041121208, "learning_rate": 6.455107527032678e-05, "loss_nats": 0.9982463121414185, "step": 6230, "unclipped_grad_norm": 0.8145909309387207 }, { "bits_per_byte": 1.458188613939892, "learning_rate": 6.417910060713926e-05, "loss_nats": 1.0107393264770508, "step": 6240, "unclipped_grad_norm": 0.8962635397911072 }, { "bits_per_byte": 1.4595658510080016, "learning_rate": 6.380884869494799e-05, "loss_nats": 1.0116939544677734, "step": 6250, "unclipped_grad_norm": 0.855821430683136 }, { "bits_per_byte": 1.5147881043388425, "learning_rate": 6.34403258603367e-05, "loss_nats": 1.049971103668213, "step": 6260, "unclipped_grad_norm": 0.8663405179977417 }, { "bits_per_byte": 1.4764731214490225, "learning_rate": 6.30735384003437e-05, "loss_nats": 1.0234131813049316, "step": 6270, "unclipped_grad_norm": 0.8501848578453064 }, { "bits_per_byte": 1.59573105103682, "learning_rate": 6.270849258235484e-05, "loss_nats": 1.1060764789581299, "step": 6280, "unclipped_grad_norm": 0.9987632036209106 }, { "bits_per_byte": 1.4579987450933396, "learning_rate": 6.234519464399581e-05, "loss_nats": 1.0106077194213867, "step": 6290, "unclipped_grad_norm": 0.8792169690132141 }, { "bits_per_byte": 1.4898052165439075, "learning_rate": 6.198365079302614e-05, "loss_nats": 1.0326542854309082, "step": 6300, "unclipped_grad_norm": 0.8601147532463074 }, { "bits_per_byte": 1.537688110249178, "learning_rate": 6.16238672072326e-05, "loss_nats": 1.065844178199768, "step": 6310, "unclipped_grad_norm": 0.907733678817749 }, { "bits_per_byte": 1.487509248154889, "learning_rate": 6.126585003432406e-05, "loss_nats": 1.0310628414154053, "step": 6320, "unclipped_grad_norm": 0.8578711152076721 }, { "bits_per_byte": 1.4237945775157679, "learning_rate": 6.090960539182614e-05, "loss_nats": 0.986899197101593, "step": 6330, "unclipped_grad_norm": 0.845673143863678 }, { "bits_per_byte": 1.4966876182661353, "learning_rate": 6.0555139366976935e-05, "loss_nats": 1.0374248027801514, "step": 6340, "unclipped_grad_norm": 0.8802160620689392 }, { "bits_per_byte": 1.485631197607467, "learning_rate": 6.020245801662274e-05, "loss_nats": 1.0297610759735107, "step": 6350, "unclipped_grad_norm": 0.884225606918335 }, { "bits_per_byte": 1.4432266392627804, "learning_rate": 5.985156736711483e-05, "loss_nats": 1.0003684759140015, "step": 6360, "unclipped_grad_norm": 0.8594306707382202 }, { "bits_per_byte": 1.497669639202562, "learning_rate": 5.950247341420618e-05, "loss_nats": 1.0381054878234863, "step": 6370, "unclipped_grad_norm": 0.9075121283531189 }, { "bits_per_byte": 1.4643473126672892, "learning_rate": 5.915518212294935e-05, "loss_nats": 1.0150082111358643, "step": 6380, "unclipped_grad_norm": 0.8724440932273865 }, { "bits_per_byte": 1.4983035672536422, "learning_rate": 5.880969942759425e-05, "loss_nats": 1.0385448932647705, "step": 6390, "unclipped_grad_norm": 0.9329665303230286 }, { "bits_per_byte": 1.4855978329731998, "learning_rate": 5.846603123148688e-05, "loss_nats": 1.029737949371338, "step": 6400, "unclipped_grad_norm": 0.8683961033821106 }, { "bits_per_byte": 1.4585226042478678, "learning_rate": 5.812418340696858e-05, "loss_nats": 1.0109708309173584, "step": 6410, "unclipped_grad_norm": 0.8944246768951416 }, { "bits_per_byte": 1.4417726979323862, "learning_rate": 5.778416179527541e-05, "loss_nats": 0.9993606805801392, "step": 6420, "unclipped_grad_norm": 0.8450288772583008 }, { "bits_per_byte": 1.474598510554618, "learning_rate": 5.7445972206438635e-05, "loss_nats": 1.0221138000488281, "step": 6430, "unclipped_grad_norm": 0.8708812594413757 }, { "bits_per_byte": 1.4773634756325393, "learning_rate": 5.710962041918516e-05, "loss_nats": 1.024030327796936, "step": 6440, "unclipped_grad_norm": 0.8653560876846313 }, { "bits_per_byte": 1.4567353605401014, "learning_rate": 5.677511218083908e-05, "loss_nats": 1.0097320079803467, "step": 6450, "unclipped_grad_norm": 0.8851431012153625 }, { "bits_per_byte": 1.4433678370191387, "learning_rate": 5.644245320722316e-05, "loss_nats": 1.0004663467407227, "step": 6460, "unclipped_grad_norm": 0.902314305305481 }, { "bits_per_byte": 1.478021997202693, "learning_rate": 5.61116491825615e-05, "loss_nats": 1.024486780166626, "step": 6470, "unclipped_grad_norm": 0.8634862899780273 }, { "bits_per_byte": 1.49569390250945, "learning_rate": 5.5782705759382104e-05, "loss_nats": 1.036736011505127, "step": 6480, "unclipped_grad_norm": 0.8877258896827698 }, { "bits_per_byte": 1.4739315618345719, "learning_rate": 5.5455628558420546e-05, "loss_nats": 1.0216515064239502, "step": 6490, "unclipped_grad_norm": 0.9130761027336121 }, { "bits_per_byte": 1.4798123365981748, "learning_rate": 5.513042316852368e-05, "loss_nats": 1.0257277488708496, "step": 6500, "unclipped_grad_norm": 0.9244931936264038 }, { "bits_per_byte": 1.429752142532981, "learning_rate": 5.480709514655443e-05, "loss_nats": 0.9910286664962769, "step": 6510, "unclipped_grad_norm": 0.8583832383155823 }, { "bits_per_byte": 1.3992985726054255, "learning_rate": 5.448565001729653e-05, "loss_nats": 0.9699198603630066, "step": 6520, "unclipped_grad_norm": 0.8968297243118286 }, { "bits_per_byte": 1.5067831718544331, "learning_rate": 5.41660932733604e-05, "loss_nats": 1.0444225072860718, "step": 6530, "unclipped_grad_norm": 0.9009160399436951 }, { "bits_per_byte": 1.468959715380779, "learning_rate": 5.384843037508902e-05, "loss_nats": 1.0182052850723267, "step": 6540, "unclipped_grad_norm": 0.8687822222709656 }, { "bits_per_byte": 1.5110841139872102, "learning_rate": 5.353266675046498e-05, "loss_nats": 1.0474036931991577, "step": 6550, "unclipped_grad_norm": 0.9212287068367004 }, { "bits_per_byte": 1.4657298811975754, "learning_rate": 5.321880779501729e-05, "loss_nats": 1.015966534614563, "step": 6560, "unclipped_grad_norm": 0.8736171126365662 }, { "bits_per_byte": 1.3923212364599213, "learning_rate": 5.290685887172962e-05, "loss_nats": 0.9650835394859314, "step": 6570, "unclipped_grad_norm": 0.9459646344184875 }, { "bits_per_byte": 1.5011586512406159, "learning_rate": 5.2596825310948295e-05, "loss_nats": 1.040523886680603, "step": 6580, "unclipped_grad_norm": 0.8902279138565063 }, { "bits_per_byte": 1.4631396504929306, "learning_rate": 5.228871241029154e-05, "loss_nats": 1.0141711235046387, "step": 6590, "unclipped_grad_norm": 0.8934556245803833 }, { "bits_per_byte": 1.3892133379761795, "learning_rate": 5.198252543455865e-05, "loss_nats": 0.9629293084144592, "step": 6600, "unclipped_grad_norm": 0.8376922607421875 }, { "bits_per_byte": 1.5024242715683152, "learning_rate": 5.167826961564027e-05, "loss_nats": 1.0414011478424072, "step": 6610, "unclipped_grad_norm": 0.8684409856796265 }, { "bits_per_byte": 1.5173028346597586, "learning_rate": 5.137595015242884e-05, "loss_nats": 1.0517141819000244, "step": 6620, "unclipped_grad_norm": 0.9338745474815369 }, { "bits_per_byte": 1.4964176055042808, "learning_rate": 5.1075572210729925e-05, "loss_nats": 1.0372376441955566, "step": 6630, "unclipped_grad_norm": 0.9484766721725464 }, { "bits_per_byte": 1.4696447222791655, "learning_rate": 5.07771409231737e-05, "loss_nats": 1.0186800956726074, "step": 6640, "unclipped_grad_norm": 0.9511991143226624 }, { "bits_per_byte": 1.4476113369465278, "learning_rate": 5.048066138912756e-05, "loss_nats": 1.0034077167510986, "step": 6650, "unclipped_grad_norm": 0.8744685649871826 }, { "bits_per_byte": 1.5102728718230902, "learning_rate": 5.018613867460863e-05, "loss_nats": 1.0468413829803467, "step": 6660, "unclipped_grad_norm": 0.9144671559333801 }, { "bits_per_byte": 1.4729612357184039, "learning_rate": 4.989357781219755e-05, "loss_nats": 1.0209789276123047, "step": 6670, "unclipped_grad_norm": 0.918013870716095 }, { "bits_per_byte": 1.4976474534406006, "learning_rate": 4.960298380095222e-05, "loss_nats": 1.0380901098251343, "step": 6680, "unclipped_grad_norm": 0.9484038352966309 }, { "bits_per_byte": 1.4621077545877539, "learning_rate": 4.931436160632256e-05, "loss_nats": 1.013455867767334, "step": 6690, "unclipped_grad_norm": 0.9366878867149353 }, { "bits_per_byte": 1.513278096664267, "learning_rate": 4.9027716160065485e-05, "loss_nats": 1.048924446105957, "step": 6700, "unclipped_grad_norm": 0.9588149785995483 }, { "bits_per_byte": 1.448871969777352, "learning_rate": 4.874305236016084e-05, "loss_nats": 1.0042815208435059, "step": 6710, "unclipped_grad_norm": 0.8966986536979675 }, { "bits_per_byte": 1.4475960304906008, "learning_rate": 4.846037507072752e-05, "loss_nats": 1.0033971071243286, "step": 6720, "unclipped_grad_norm": 0.9147130250930786 }, { "bits_per_byte": 1.4208115384365525, "learning_rate": 4.817968912194049e-05, "loss_nats": 0.9848315119743347, "step": 6730, "unclipped_grad_norm": 0.9434075951576233 }, { "bits_per_byte": 1.4530764296429952, "learning_rate": 4.790099930994813e-05, "loss_nats": 1.0071958303451538, "step": 6740, "unclipped_grad_norm": 0.9062299728393555 }, { "bits_per_byte": 1.4277258429405155, "learning_rate": 4.762431039679047e-05, "loss_nats": 0.9896241426467896, "step": 6750, "unclipped_grad_norm": 0.9092971086502075 }, { "bits_per_byte": 1.4128754850038137, "learning_rate": 4.734962711031755e-05, "loss_nats": 0.9793306589126587, "step": 6760, "unclipped_grad_norm": 0.8910818099975586 }, { "bits_per_byte": 1.434429210723195, "learning_rate": 4.707695414410892e-05, "loss_nats": 0.9942705631256104, "step": 6770, "unclipped_grad_norm": 0.8744387030601501 }, { "bits_per_byte": 1.4315308730995298, "learning_rate": 4.680629615739322e-05, "loss_nats": 0.9922615885734558, "step": 6780, "unclipped_grad_norm": 0.9047504663467407 }, { "bits_per_byte": 1.53544803622169, "learning_rate": 4.653765777496863e-05, "loss_nats": 1.0642914772033691, "step": 6790, "unclipped_grad_norm": 0.9123643636703491 }, { "bits_per_byte": 1.504298710480069, "learning_rate": 4.6271043587123986e-05, "loss_nats": 1.0427004098892212, "step": 6800, "unclipped_grad_norm": 0.978726327419281 }, { "bits_per_byte": 1.4819994399691971, "learning_rate": 4.600645814956006e-05, "loss_nats": 1.027243733406067, "step": 6810, "unclipped_grad_norm": 0.9510568380355835 }, { "bits_per_byte": 1.4201767504722178, "learning_rate": 4.574390598331204e-05, "loss_nats": 0.9843915104866028, "step": 6820, "unclipped_grad_norm": 0.9054594039916992 }, { "bits_per_byte": 1.441129998766112, "learning_rate": 4.548339157467199e-05, "loss_nats": 0.9989151954650879, "step": 6830, "unclipped_grad_norm": 0.9485244154930115 }, { "bits_per_byte": 1.4736460706341397, "learning_rate": 4.52249193751124e-05, "loss_nats": 1.021453619003296, "step": 6840, "unclipped_grad_norm": 0.9141860604286194 }, { "bits_per_byte": 1.439767810179953, "learning_rate": 4.4968493801209985e-05, "loss_nats": 0.9979709982872009, "step": 6850, "unclipped_grad_norm": 0.9174992442131042 }, { "bits_per_byte": 1.4496288654237992, "learning_rate": 4.471411923457034e-05, "loss_nats": 1.0048061609268188, "step": 6860, "unclipped_grad_norm": 0.9317811727523804 }, { "bits_per_byte": 1.4588201342338605, "learning_rate": 4.446180002175291e-05, "loss_nats": 1.0111770629882812, "step": 6870, "unclipped_grad_norm": 0.9191097617149353 }, { "bits_per_byte": 1.4995917173086046, "learning_rate": 4.421154047419693e-05, "loss_nats": 1.0394377708435059, "step": 6880, "unclipped_grad_norm": 0.9317643642425537 }, { "bits_per_byte": 1.3990970949299397, "learning_rate": 4.396334486814753e-05, "loss_nats": 0.9697802066802979, "step": 6890, "unclipped_grad_norm": 0.9130836725234985 }, { "bits_per_byte": 1.4230204836042344, "learning_rate": 4.371721744458285e-05, "loss_nats": 0.986362636089325, "step": 6900, "unclipped_grad_norm": 0.8910167217254639 }, { "bits_per_byte": 1.5914774041330302, "learning_rate": 4.347316240914145e-05, "loss_nats": 1.1031280755996704, "step": 6910, "unclipped_grad_norm": 1.0252712965011597 }, { "bits_per_byte": 1.394897880535148, "learning_rate": 4.3231183932050546e-05, "loss_nats": 0.9668695330619812, "step": 6920, "unclipped_grad_norm": 0.8750832676887512 }, { "bits_per_byte": 1.42465509270936, "learning_rate": 4.299128614805464e-05, "loss_nats": 0.9874956607818604, "step": 6930, "unclipped_grad_norm": 0.8865536451339722 }, { "bits_per_byte": 1.4536008047454758, "learning_rate": 4.2753473156344965e-05, "loss_nats": 1.0075592994689941, "step": 6940, "unclipped_grad_norm": 0.913940966129303 }, { "bits_per_byte": 1.4401447961506444, "learning_rate": 4.251774902048942e-05, "loss_nats": 0.9982323050498962, "step": 6950, "unclipped_grad_norm": 0.941871702671051 }, { "bits_per_byte": 1.4963969675861772, "learning_rate": 4.228411776836306e-05, "loss_nats": 1.0372233390808105, "step": 6960, "unclipped_grad_norm": 0.9974261522293091 }, { "bits_per_byte": 1.4704031657694705, "learning_rate": 4.2052583392079427e-05, "loss_nats": 1.0192058086395264, "step": 6970, "unclipped_grad_norm": 0.9634589552879333 }, { "bits_per_byte": 1.5157009882496222, "learning_rate": 4.182314984792214e-05, "loss_nats": 1.0506038665771484, "step": 6980, "unclipped_grad_norm": 0.9805173873901367 }, { "bits_per_byte": 1.4572141602401034, "learning_rate": 4.159582105627748e-05, "loss_nats": 1.010063886642456, "step": 6990, "unclipped_grad_norm": 0.9425977468490601 }, { "bits_per_byte": 1.4146713278441232, "learning_rate": 4.137060090156724e-05, "loss_nats": 0.980575442314148, "step": 7000, "unclipped_grad_norm": 0.9211949110031128 }, { "bits_per_byte": 1.4423556331274856, "learning_rate": 4.114749323218249e-05, "loss_nats": 0.9997647404670715, "step": 7010, "unclipped_grad_norm": 0.9508169293403625 }, { "bits_per_byte": 1.4268384124620634, "learning_rate": 4.0926501860417706e-05, "loss_nats": 0.9890090227127075, "step": 7020, "unclipped_grad_norm": 0.9370734095573425 }, { "bits_per_byte": 1.4214180352548202, "learning_rate": 4.070763056240571e-05, "loss_nats": 0.9852519035339355, "step": 7030, "unclipped_grad_norm": 0.8771427273750305 }, { "bits_per_byte": 1.5093840654834314, "learning_rate": 4.049088307805306e-05, "loss_nats": 1.0462253093719482, "step": 7040, "unclipped_grad_norm": 0.9992957711219788 }, { "bits_per_byte": 1.466185635222362, "learning_rate": 4.027626311097629e-05, "loss_nats": 1.0162824392318726, "step": 7050, "unclipped_grad_norm": 0.9372990131378174 }, { "bits_per_byte": 1.4783773133593756, "learning_rate": 4.006377432843846e-05, "loss_nats": 1.024733066558838, "step": 7060, "unclipped_grad_norm": 1.0182687044143677 }, { "bits_per_byte": 1.4142496263842075, "learning_rate": 3.985342036128656e-05, "loss_nats": 0.9802831411361694, "step": 7070, "unclipped_grad_norm": 0.8880259394645691 }, { "bits_per_byte": 1.3447492013960853, "learning_rate": 3.964520480388956e-05, "loss_nats": 0.9321091175079346, "step": 7080, "unclipped_grad_norm": 0.910698413848877 }, { "bits_per_byte": 1.413691198716856, "learning_rate": 3.943913121407678e-05, "loss_nats": 0.979896068572998, "step": 7090, "unclipped_grad_norm": 0.9244340062141418 }, { "bits_per_byte": 1.4025504205679395, "learning_rate": 3.923520311307735e-05, "loss_nats": 0.9721738696098328, "step": 7100, "unclipped_grad_norm": 0.9016135334968567 }, { "bits_per_byte": 1.4533139376838367, "learning_rate": 3.9033423985459816e-05, "loss_nats": 1.0073604583740234, "step": 7110, "unclipped_grad_norm": 0.970280110836029 }, { "bits_per_byte": 1.4606615524766484, "learning_rate": 3.88337972790728e-05, "loss_nats": 1.0124534368515015, "step": 7120, "unclipped_grad_norm": 0.9771916270256042 }, { "bits_per_byte": 1.396041221198084, "learning_rate": 3.8636326404985845e-05, "loss_nats": 0.9676620364189148, "step": 7130, "unclipped_grad_norm": 0.8866344094276428 }, { "bits_per_byte": 1.5095894127685614, "learning_rate": 3.844101473743143e-05, "loss_nats": 1.0463676452636719, "step": 7140, "unclipped_grad_norm": 0.9947805404663086 }, { "bits_per_byte": 1.5291790966150898, "learning_rate": 3.824786561374701e-05, "loss_nats": 1.0599461793899536, "step": 7150, "unclipped_grad_norm": 0.9418985247612 }, { "bits_per_byte": 1.4640222654571586, "learning_rate": 3.805688233431824e-05, "loss_nats": 1.0147829055786133, "step": 7160, "unclipped_grad_norm": 1.0358314514160156 }, { "bits_per_byte": 1.4559156912260893, "learning_rate": 3.786806816252236e-05, "loss_nats": 1.0091638565063477, "step": 7170, "unclipped_grad_norm": 0.9539772272109985 }, { "bits_per_byte": 1.3591113005869877, "learning_rate": 3.768142632467269e-05, "loss_nats": 0.9420641660690308, "step": 7180, "unclipped_grad_norm": 0.8915972709655762 }, { "bits_per_byte": 1.3840849873101015, "learning_rate": 3.7496960009963236e-05, "loss_nats": 0.9593746066093445, "step": 7190, "unclipped_grad_norm": 0.9118176102638245 }, { "bits_per_byte": 1.4715360154907038, "learning_rate": 3.7314672370414325e-05, "loss_nats": 1.0199910402297974, "step": 7200, "unclipped_grad_norm": 0.973518431186676 }, { "bits_per_byte": 1.4678313372084681, "learning_rate": 3.713456652081882e-05, "loss_nats": 1.017423152923584, "step": 7210, "unclipped_grad_norm": 0.9181845188140869 }, { "bits_per_byte": 1.4265923052886786, "learning_rate": 3.695664553868873e-05, "loss_nats": 0.9888384342193604, "step": 7220, "unclipped_grad_norm": 0.926552951335907 }, { "bits_per_byte": 1.4337483314084294, "learning_rate": 3.678091246420277e-05, "loss_nats": 0.9937986135482788, "step": 7230, "unclipped_grad_norm": 0.9526586532592773 }, { "bits_per_byte": 1.383381664259398, "learning_rate": 3.660737030015427e-05, "loss_nats": 0.9588871002197266, "step": 7240, "unclipped_grad_norm": 0.8890237212181091 }, { "bits_per_byte": 1.431437056563484, "learning_rate": 3.6436022011900056e-05, "loss_nats": 0.9921965599060059, "step": 7250, "unclipped_grad_norm": 0.9829798340797424 }, { "bits_per_byte": 1.4272578781475178, "learning_rate": 3.626687052730954e-05, "loss_nats": 0.9892997741699219, "step": 7260, "unclipped_grad_norm": 0.9537968039512634 }, { "bits_per_byte": 1.5069463833901018, "learning_rate": 3.6099918736714955e-05, "loss_nats": 1.0445356369018555, "step": 7270, "unclipped_grad_norm": 0.9953339099884033 }, { "bits_per_byte": 1.4318632295723221, "learning_rate": 3.5935169492861716e-05, "loss_nats": 0.9924919605255127, "step": 7280, "unclipped_grad_norm": 0.9815621972084045 }, { "bits_per_byte": 1.435238647069481, "learning_rate": 3.5772625610859886e-05, "loss_nats": 0.9948316216468811, "step": 7290, "unclipped_grad_norm": 0.9373346567153931 }, { "bits_per_byte": 1.44098209368637, "learning_rate": 3.5612289868135887e-05, "loss_nats": 0.9988126754760742, "step": 7300, "unclipped_grad_norm": 0.9782799482345581 }, { "bits_per_byte": 1.467732619166873, "learning_rate": 3.545416500438522e-05, "loss_nats": 1.0173547267913818, "step": 7310, "unclipped_grad_norm": 0.9545702338218689 }, { "bits_per_byte": 1.4680886232541588, "learning_rate": 3.52982537215255e-05, "loss_nats": 1.017601490020752, "step": 7320, "unclipped_grad_norm": 0.9820490479469299 }, { "bits_per_byte": 1.4597027491980883, "learning_rate": 3.5144558683650396e-05, "loss_nats": 1.0117888450622559, "step": 7330, "unclipped_grad_norm": 1.0537965297698975 }, { "bits_per_byte": 1.4358094574876945, "learning_rate": 3.499308251698404e-05, "loss_nats": 0.9952272772789001, "step": 7340, "unclipped_grad_norm": 0.9470505714416504 }, { "bits_per_byte": 1.4763711357370608, "learning_rate": 3.484382780983616e-05, "loss_nats": 1.023342490196228, "step": 7350, "unclipped_grad_norm": 0.93792724609375 }, { "bits_per_byte": 1.4524729425211176, "learning_rate": 3.469679711255795e-05, "loss_nats": 1.0067775249481201, "step": 7360, "unclipped_grad_norm": 0.959013044834137 }, { "bits_per_byte": 1.4678055398108387, "learning_rate": 3.4551992937498284e-05, "loss_nats": 1.0174052715301514, "step": 7370, "unclipped_grad_norm": 0.9769075512886047 }, { "bits_per_byte": 1.5064952728968888, "learning_rate": 3.440941775896104e-05, "loss_nats": 1.0442229509353638, "step": 7380, "unclipped_grad_norm": 1.0261726379394531 }, { "bits_per_byte": 1.4743861119808024, "learning_rate": 3.4269074013162596e-05, "loss_nats": 1.021966576576233, "step": 7390, "unclipped_grad_norm": 0.977978527545929 }, { "bits_per_byte": 1.4106443540741704, "learning_rate": 3.41309640981904e-05, "loss_nats": 0.9777841567993164, "step": 7400, "unclipped_grad_norm": 0.9216173887252808 }, { "bits_per_byte": 1.4343313525948542, "learning_rate": 3.3995090373961795e-05, "loss_nats": 0.994202733039856, "step": 7410, "unclipped_grad_norm": 0.9778451919555664 }, { "bits_per_byte": 1.5075760118749106, "learning_rate": 3.3861455162183876e-05, "loss_nats": 1.0449720621109009, "step": 7420, "unclipped_grad_norm": 1.0143206119537354 }, { "bits_per_byte": 1.4176087055295346, "learning_rate": 3.3730060746313664e-05, "loss_nats": 0.9826114773750305, "step": 7430, "unclipped_grad_norm": 0.9865745902061462 }, { "bits_per_byte": 1.4384498211350658, "learning_rate": 3.3600909371519225e-05, "loss_nats": 0.9970574378967285, "step": 7440, "unclipped_grad_norm": 0.9753794074058533 }, { "bits_per_byte": 1.385612537215065, "learning_rate": 3.3474003244641194e-05, "loss_nats": 0.9604334235191345, "step": 7450, "unclipped_grad_norm": 0.9882722496986389 }, { "bits_per_byte": 1.40335280562554, "learning_rate": 3.334934453415514e-05, "loss_nats": 0.9727300405502319, "step": 7460, "unclipped_grad_norm": 0.8938714861869812 }, { "bits_per_byte": 1.4650283639647061, "learning_rate": 3.3226935370134486e-05, "loss_nats": 1.0154802799224854, "step": 7470, "unclipped_grad_norm": 0.95097815990448 }, { "bits_per_byte": 1.378530205661209, "learning_rate": 3.31067778442141e-05, "loss_nats": 0.9555243253707886, "step": 7480, "unclipped_grad_norm": 0.9593113660812378 }, { "bits_per_byte": 1.4079058743244819, "learning_rate": 3.298887400955457e-05, "loss_nats": 0.9758859872817993, "step": 7490, "unclipped_grad_norm": 0.9498793482780457 }, { "bits_per_byte": 1.4795925427703722, "learning_rate": 3.287322588080714e-05, "loss_nats": 1.0255753993988037, "step": 7500, "unclipped_grad_norm": 0.9999135732650757 }, { "bits_per_byte": 1.462181535144974, "learning_rate": 3.275983543407924e-05, "loss_nats": 1.0135070085525513, "step": 7510, "unclipped_grad_norm": 0.96773761510849 }, { "bits_per_byte": 1.424363066168195, "learning_rate": 3.2648704606900735e-05, "loss_nats": 0.9872932434082031, "step": 7520, "unclipped_grad_norm": 0.9349227547645569 }, { "bits_per_byte": 1.4575968216382733, "learning_rate": 3.253983529819086e-05, "loss_nats": 1.0103291273117065, "step": 7530, "unclipped_grad_norm": 0.9571959972381592 }, { "bits_per_byte": 1.484410464751643, "learning_rate": 3.243322936822571e-05, "loss_nats": 1.0289149284362793, "step": 7540, "unclipped_grad_norm": 0.9889418482780457 }, { "bits_per_byte": 1.4190770074112757, "learning_rate": 3.232888863860648e-05, "loss_nats": 0.9836292266845703, "step": 7550, "unclipped_grad_norm": 0.9660657048225403 }, { "bits_per_byte": 1.458957376389249, "learning_rate": 3.222681489222838e-05, "loss_nats": 1.0112721920013428, "step": 7560, "unclipped_grad_norm": 0.9621381163597107 }, { "bits_per_byte": 1.4744461339259534, "learning_rate": 3.212700987325009e-05, "loss_nats": 1.0220081806182861, "step": 7570, "unclipped_grad_norm": 0.9809478521347046 }, { "bits_per_byte": 1.424117474942763, "learning_rate": 3.202947528706403e-05, "loss_nats": 0.9871230125427246, "step": 7580, "unclipped_grad_norm": 1.0034832954406738 }, { "bits_per_byte": 1.4167208450944553, "learning_rate": 3.1934212800267116e-05, "loss_nats": 0.9819960594177246, "step": 7590, "unclipped_grad_norm": 0.9541960954666138 }, { "bits_per_byte": 1.3862604818521906, "learning_rate": 3.1841224040632484e-05, "loss_nats": 0.9608825445175171, "step": 7600, "unclipped_grad_norm": 0.950609564781189 }, { "bits_per_byte": 1.3616721222583348, "learning_rate": 3.175051059708143e-05, "loss_nats": 0.9438391923904419, "step": 7610, "unclipped_grad_norm": 0.9224535226821899 }, { "bits_per_byte": 1.4445553772233464, "learning_rate": 3.1662074019656414e-05, "loss_nats": 1.0012894868850708, "step": 7620, "unclipped_grad_norm": 0.9717100858688354 }, { "bits_per_byte": 1.4175787805482842, "learning_rate": 3.157591581949455e-05, "loss_nats": 0.9825907349586487, "step": 7630, "unclipped_grad_norm": 0.9714697599411011 }, { "bits_per_byte": 1.4152020663046858, "learning_rate": 3.149203746880175e-05, "loss_nats": 0.9809433221817017, "step": 7640, "unclipped_grad_norm": 0.9795277118682861 }, { "bits_per_byte": 1.5113582543326856, "learning_rate": 3.141044040082763e-05, "loss_nats": 1.0475937128067017, "step": 7650, "unclipped_grad_norm": 1.0563151836395264 }, { "bits_per_byte": 1.4375178471500403, "learning_rate": 3.133112600984091e-05, "loss_nats": 0.9964114427566528, "step": 7660, "unclipped_grad_norm": 0.9650751948356628 }, { "bits_per_byte": 1.4172367070557181, "learning_rate": 3.125409565110571e-05, "loss_nats": 0.9823536276817322, "step": 7670, "unclipped_grad_norm": 0.9894714951515198 }, { "bits_per_byte": 1.3937812831744212, "learning_rate": 3.117935064085835e-05, "loss_nats": 0.9660955667495728, "step": 7680, "unclipped_grad_norm": 0.9648637175559998 }, { "bits_per_byte": 1.4266868097553278, "learning_rate": 3.110689225628479e-05, "loss_nats": 0.9889039397239685, "step": 7690, "unclipped_grad_norm": 0.9686183333396912 }, { "bits_per_byte": 1.4503658110827464, "learning_rate": 3.1036721735498926e-05, "loss_nats": 1.005316972732544, "step": 7700, "unclipped_grad_norm": 0.9993582367897034 }, { "bits_per_byte": 1.3691781330725912, "learning_rate": 3.096884027752132e-05, "loss_nats": 0.9490419626235962, "step": 7710, "unclipped_grad_norm": 0.9620972275733948 }, { "bits_per_byte": 1.3650053180147068, "learning_rate": 3.090324904225885e-05, "loss_nats": 0.9461495876312256, "step": 7720, "unclipped_grad_norm": 0.9128993153572083 }, { "bits_per_byte": 1.4252105106803215, "learning_rate": 3.083994915048472e-05, "loss_nats": 0.9878806471824646, "step": 7730, "unclipped_grad_norm": 0.9594771265983582 }, { "bits_per_byte": 1.4081796707046554, "learning_rate": 3.077894168381942e-05, "loss_nats": 0.9760757684707642, "step": 7740, "unclipped_grad_norm": 0.9255442023277283 }, { "bits_per_byte": 1.433292233418341, "learning_rate": 3.0720227684712254e-05, "loss_nats": 0.9934824705123901, "step": 7750, "unclipped_grad_norm": 0.9785766005516052 }, { "bits_per_byte": 1.3999133245909345, "learning_rate": 3.0663808156423456e-05, "loss_nats": 0.9703459739685059, "step": 7760, "unclipped_grad_norm": 1.0032256841659546 }, { "bits_per_byte": 1.4048038232508695, "learning_rate": 3.060968406300711e-05, "loss_nats": 0.9737358093261719, "step": 7770, "unclipped_grad_norm": 1.0204466581344604 }, { "bits_per_byte": 1.536527399338505, "learning_rate": 3.055785632929463e-05, "loss_nats": 1.0650396347045898, "step": 7780, "unclipped_grad_norm": 1.0556561946868896 }, { "bits_per_byte": 1.4128328333063997, "learning_rate": 3.0508325840878984e-05, "loss_nats": 0.9793010950088501, "step": 7790, "unclipped_grad_norm": 1.0078489780426025 }, { "bits_per_byte": 1.451735308931567, "learning_rate": 3.046109344409957e-05, "loss_nats": 1.0062662363052368, "step": 7800, "unclipped_grad_norm": 1.0289596319198608 }, { "bits_per_byte": 1.3852072600983067, "learning_rate": 3.0416159946027727e-05, "loss_nats": 0.9601525068283081, "step": 7810, "unclipped_grad_norm": 0.9891387224197388 }, { "bits_per_byte": 1.3953124447150527, "learning_rate": 3.0373526114452946e-05, "loss_nats": 0.9671568870544434, "step": 7820, "unclipped_grad_norm": 0.9778857231140137 }, { "bits_per_byte": 1.45392653988621, "learning_rate": 3.0333192677869803e-05, "loss_nats": 1.0077850818634033, "step": 7830, "unclipped_grad_norm": 1.0185887813568115 }, { "bits_per_byte": 1.4113820736550464, "learning_rate": 3.029516032546544e-05, "loss_nats": 0.9782955050468445, "step": 7840, "unclipped_grad_norm": 0.9330953359603882 }, { "bits_per_byte": 1.380841996454107, "learning_rate": 3.02594297071078e-05, "loss_nats": 0.9571267366409302, "step": 7850, "unclipped_grad_norm": 0.9742368459701538 }, { "bits_per_byte": 1.3846048048723343, "learning_rate": 3.0226001433334603e-05, "loss_nats": 0.9597349166870117, "step": 7860, "unclipped_grad_norm": 0.9645414352416992 }, { "bits_per_byte": 1.4429803601067448, "learning_rate": 3.019487607534278e-05, "loss_nats": 1.0001977682113647, "step": 7870, "unclipped_grad_norm": 0.9958038926124573 }, { "bits_per_byte": 1.4256185395194934, "learning_rate": 3.016605416497886e-05, "loss_nats": 0.9881634712219238, "step": 7880, "unclipped_grad_norm": 0.9739922881126404 }, { "bits_per_byte": 1.4834710955126302, "learning_rate": 3.0139536194729734e-05, "loss_nats": 1.028263807296753, "step": 7890, "unclipped_grad_norm": 0.9912284016609192 }, { "bits_per_byte": 1.4598313922209336, "learning_rate": 3.0115322617714362e-05, "loss_nats": 1.0118780136108398, "step": 7900, "unclipped_grad_norm": 0.9979069232940674 }, { "bits_per_byte": 1.465345499972897, "learning_rate": 3.0093413847675965e-05, "loss_nats": 1.015700101852417, "step": 7910, "unclipped_grad_norm": 1.0503500699996948 }, { "bits_per_byte": 1.3660396216769959, "learning_rate": 3.0073810258974985e-05, "loss_nats": 0.946866512298584, "step": 7920, "unclipped_grad_norm": 0.9824531078338623 }, { "bits_per_byte": 1.3848080023743288, "learning_rate": 3.0056512186582658e-05, "loss_nats": 0.959875762462616, "step": 7930, "unclipped_grad_norm": 1.0091050863265991 }, { "bits_per_byte": 1.4375059803471308, "learning_rate": 3.0041519926075316e-05, "loss_nats": 0.9964032173156738, "step": 7940, "unclipped_grad_norm": 1.0658900737762451 }, { "bits_per_byte": 1.4049340141175726, "learning_rate": 3.0028833733629313e-05, "loss_nats": 0.9738260507583618, "step": 7950, "unclipped_grad_norm": 1.0148085355758667 }, { "bits_per_byte": 1.415199744538899, "learning_rate": 3.0018453826016686e-05, "loss_nats": 0.9809417128562927, "step": 7960, "unclipped_grad_norm": 0.9737400412559509 }, { "bits_per_byte": 1.4008050546356583, "learning_rate": 3.001038038060143e-05, "loss_nats": 0.9709640741348267, "step": 7970, "unclipped_grad_norm": 0.9626508951187134 }, { "bits_per_byte": 1.4372590132604919, "learning_rate": 3.0004613535336433e-05, "loss_nats": 0.9962320327758789, "step": 7980, "unclipped_grad_norm": 0.9673776626586914 }, { "bits_per_byte": 1.4403348369798477, "learning_rate": 3.000115338876117e-05, "loss_nats": 0.9983640313148499, "step": 7990, "unclipped_grad_norm": 0.9839751720428467 }, { "bits_per_byte": 1.3635582559903472, "learning_rate": 3e-05, "loss_nats": 0.9451465606689453, "step": 8000, "unclipped_grad_norm": 0.9773924946784973 } ] }